05:37:53.852 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
05:37:53.864 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasks6jlpt/test-tmp
05:37:53.867 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:37:53.870 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasks6jlpt/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:37:54.138 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
05:37:54.286 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:37:54.287 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:37:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:54.291177 9821 mini_oidc.cc:152] Starting JWKS server
05:37:54.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:54.295293 9821 webserver.cc:460] Webserver started at http://127.0.0.1:40497/ using document root <none> and password file <none>
05:37:54.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:54.295840 9821 mini_oidc.cc:189] Starting OIDC Discovery server
05:37:54.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:54.297648 9821 webserver.cc:460] Webserver started at http://127.0.0.1:42321/ using document root <none> and password file <none>
05:37:54.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:37:54Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:37:54.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:37:54Z Disabled control of system clock
05:37:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:54.350746 9821 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:37:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:37:54.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/wal
05:37:54.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/data
05:37:54.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:37:54.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:37:54.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:37:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:37:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:37:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:37:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/logs
05:37:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/data/info.pb
05:37:54.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:37:54.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:37:54.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:37:54.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:37:54.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:37:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:37:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:37:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:37:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:37:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:37:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:37:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:37:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.151.126:43541
05:37:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.151.126
05:37:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:37:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.151.84:42897
05:37:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:37:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:37:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:37:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:37:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.151.126:43541,127.9.151.125:45533,127.9.151.124:43295
05:37:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:37:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40497/jwks.json with env {}
05:37:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:54.656468 9833 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:37:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:54.657017 9833 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:37:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:54.657635 9833 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:37:54.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:54.687827 9833 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:37:54.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:54.688139 9833 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:37:54.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:54.688407 9833 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:37:54.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:54.688596 9833 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:37:54.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:54.688887 9833 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:37:54.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:54.689121 9833 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40497/jwks.json
05:37:54.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:54.723759 9833 master_runner.cc:386] Master server non-default flags:
05:37:54.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:37:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.151.84:42897
05:37:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:37:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:37:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/data
05:37:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/wal
05:37:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:37:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.151.126:43541,127.9.151.125:45533,127.9.151.124:43295
05:37:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:37:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:37:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:37:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.151.126:43541
05:37:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:37:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:37:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:37:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40497/jwks.json
05:37:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:37:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:37:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/data/info.pb
05:37:54.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.151.126
05:37:54.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:37:54.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:37:54.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:37:54.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:37:54.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:37:54.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:37:54.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/logs
05:37:54.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:37:54.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:37:54.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:37:54.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:37:54.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:37:54.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:37:54.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:37:54.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:37:54.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:37:54.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:37:54.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:54.725325 9833 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:37:54.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:54.726867 9833 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:37:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:54.744675 9840 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:37:55.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:54.745560 9842 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:37:55.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:54.745368 9839 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:37:55.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:55.939388 9841 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:37:55.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.939522 9833 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:37:55.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.943149 9833 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:37:55.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.946528 9833 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:37:55.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.948014 9833 hybrid_clock.cc:648] HybridClock initialized: now 1782020275947975 us; error 65 us; skew 500 ppm
05:37:55.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.948807 9833 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:37:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.960475 9833 webserver.cc:460] Webserver started at http://127.9.151.126:34611/ using document root <none> and password file <none>
05:37:55.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.961489 9833 fs_manager.cc:359] Metadata directory not provided
05:37:55.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.961688 9833 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:37:55.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.962091 9833 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:37:55.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.967008 9833 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/data/instance:
05:37:55.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7ce5533f44942cd99915c72b0f83f84"
05:37:55.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:37:55 on dist-test-slave-wcx4"
05:37:55.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.968222 9833 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/wal/instance:
05:37:55.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7ce5533f44942cd99915c72b0f83f84"
05:37:55.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:37:55 on dist-test-slave-wcx4"
05:37:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.975562 9833 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
05:37:55.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.981596 9849 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:37:55.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.982787 9833 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
05:37:55.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.983156 9833 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/data,/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/wal
05:37:55.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7ce5533f44942cd99915c72b0f83f84"
05:37:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:37:55 on dist-test-slave-wcx4"
05:37:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:55.983549 9833 fs_report.cc:383] FS layout report
05:37:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:37:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/wal
05:37:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/wal
05:37:55.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/data/data
05:37:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:37:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:37:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:37:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:37:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:37:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:37:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:37:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:37:55.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:37:55.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:37:55.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:37:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.023095 9833 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:37:56.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.024822 9833 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:37:56.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.025362 9833 kserver.cc:163] Server-wide thread pool size limit: 3276
05:37:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.099773 9833 rpc_server.cc:307] RPC server started. Bound to: 127.9.151.126:43541
05:37:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.099869 9900 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.151.126:43541 every 8 connection(s)
05:37:56.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.102653 9833 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/data/info.pb
05:37:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.105146 9821 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu as pid 9833
05:37:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.106138 9821 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-0/wal/instance
05:37:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.108695 9901 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:37:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.109639 9821 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:37:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:37:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/wal
05:37:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/data
05:37:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:37:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:37:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:37:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:37:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:37:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:37:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/logs
05:37:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/data/info.pb
05:37:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:37:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:37:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:37:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:37:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:37:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:37:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:37:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:37:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:37:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:37:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:37:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:37:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.151.125:45533
05:37:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.151.125
05:37:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:37:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.151.84:42897
05:37:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:37:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:37:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:37:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:37:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.151.126:43541,127.9.151.125:45533,127.9.151.124:43295
05:37:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:37:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40497/jwks.json with env {}
05:37:56.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.121273 9901 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } has no permanent_uuid. Determining permanent_uuid...
05:37:56.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.151178 9901 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } has no permanent_uuid. Determining permanent_uuid...
05:37:56.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.153441 9852 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.151.125:45533: connect: Connection refused (error 111)
05:37:56.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.157621 9901 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.125:45533: Network error: Client connection negotiation failed: client connection to 127.9.151.125:45533: connect: Connection refused (error 111)
05:37:56.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.207054 9901 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } attempt: 1
05:37:56.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.210793 9901 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.125:45533: Network error: Client connection negotiation failed: client connection to 127.9.151.125:45533: connect: Connection refused (error 111)
05:37:56.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.279747 9901 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } attempt: 2
05:37:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.283874 9901 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.125:45533: Network error: Client connection negotiation failed: client connection to 127.9.151.125:45533: connect: Connection refused (error 111)
05:37:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.375367 9901 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } attempt: 3
05:37:56.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.379184 9901 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.125:45533: Network error: Client connection negotiation failed: client connection to 127.9.151.125:45533: connect: Connection refused (error 111)
05:37:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.430648 9903 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:37:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.431185 9903 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:37:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.431970 9903 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:37:56.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.461809 9903 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:37:56.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.462090 9903 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:37:56.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.462275 9903 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:37:56.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.462462 9903 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:37:56.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.462683 9903 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:37:56.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.462888 9903 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40497/jwks.json
05:37:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.498036 9903 master_runner.cc:386] Master server non-default flags:
05:37:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:37:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.151.84:42897
05:37:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:37:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:37:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/data
05:37:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/wal
05:37:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:37:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.151.126:43541,127.9.151.125:45533,127.9.151.124:43295
05:37:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:37:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:37:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:37:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.151.125:45533
05:37:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:37:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:37:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:37:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40497/jwks.json
05:37:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:37:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:37:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/data/info.pb
05:37:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.151.125
05:37:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:37:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:37:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:37:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:37:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:37:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:37:56.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/logs
05:37:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:37:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:37:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:37:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:37:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:37:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:37:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:37:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:37:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:37:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:37:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.499249 9903 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:37:56.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.501418 9903 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:37:56.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.521413 9912 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:37:56.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.522691 9901 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } attempt: 4
05:37:56.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.526697 9901 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.125:45533: Network error: Client connection negotiation failed: client connection to 127.9.151.125:45533: connect: Connection refused (error 111)
05:37:56.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.826104 9901 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } attempt: 5
05:37:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.829612 9901 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.125:45533: Network error: Client connection negotiation failed: client connection to 127.9.151.125:45533: connect: Connection refused (error 111)
05:37:57.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.377238 9901 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } attempt: 6
05:37:57.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:57.389402 9901 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.125:45533: Network error: Client connection negotiation failed: client connection to 127.9.151.125:45533: connect: Connection refused (error 111)
05:37:57.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.521593 9913 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:37:57.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:56.522670 9915 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:37:57.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:56.524421 9903 server_base.cc:1034] running on GCE node
05:37:57.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.714474 9903 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:37:57.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.717784 9903 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:37:57.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.719261 9903 hybrid_clock.cc:648] HybridClock initialized: now 1782020277719233 us; error 60 us; skew 500 ppm
05:37:57.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.720161 9903 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:37:57.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.727224 9903 webserver.cc:460] Webserver started at http://127.9.151.125:41523/ using document root <none> and password file <none>
05:37:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.728197 9903 fs_manager.cc:359] Metadata directory not provided
05:37:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.728392 9903 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:37:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.728765 9903 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:37:57.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.733664 9903 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/data/instance:
05:37:57.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc3ccda9f9d34b04ac8780807475e023"
05:37:57.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:37:57 on dist-test-slave-wcx4"
05:37:57.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.734825 9903 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/wal/instance:
05:37:57.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc3ccda9f9d34b04ac8780807475e023"
05:37:57.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:37:57 on dist-test-slave-wcx4"
05:37:57.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.742558 9903 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
05:37:57.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.748039 9924 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:37:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.749135 9903 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
05:37:57.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.749497 9903 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/data,/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/wal
05:37:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc3ccda9f9d34b04ac8780807475e023"
05:37:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:37:57 on dist-test-slave-wcx4"
05:37:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.749873 9903 fs_report.cc:383] FS layout report
05:37:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/wal
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/wal
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/data/data
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:37:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:37:57.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.786788 9903 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:37:57.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.788453 9903 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:37:57.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.788986 9903 kserver.cc:163] Server-wide thread pool size limit: 3276
05:37:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.863052 9903 rpc_server.cc:307] RPC server started. Bound to: 127.9.151.125:45533
05:37:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.863153 9975 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.151.125:45533 every 8 connection(s)
05:37:57.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.865921 9903 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/data/info.pb
05:37:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.872219 9976 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:37:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.876314 9821 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu as pid 9903
05:37:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.876649 9821 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-1/wal/instance
05:37:57.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.878523 9821 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/wal
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/data
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/logs
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/data/info.pb
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:37:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:37:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:37:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:37:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:37:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:37:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:37:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:37:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:37:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:37:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:37:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.151.124:43295
05:37:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.151.124
05:37:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:37:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.151.84:42897
05:37:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:37:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:37:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:37:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:37:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.151.126:43541,127.9.151.125:45533,127.9.151.124:43295
05:37:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:37:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40497/jwks.json with env {}
05:37:57.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.885178 9976 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } has no permanent_uuid. Determining permanent_uuid...
05:37:57.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.908900 9976 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } has no permanent_uuid. Determining permanent_uuid...
05:37:57.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.926435 9976 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } has no permanent_uuid. Determining permanent_uuid...
05:37:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:57.929369 9927 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:57.932869 9976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.124:43295: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:57.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:57.982357 9976 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } attempt: 1
05:37:57.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:57.985858 9976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.124:43295: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:58.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:58.054255 9976 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } attempt: 2
05:37:58.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.058444 9976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.124:43295: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:58.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:58.149825 9976 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } attempt: 3
05:37:58.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.153422 9976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.124:43295: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:58.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.191145 9978 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:37:58.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.191680 9978 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:37:58.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.192175 9978 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:37:58.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.221951 9978 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:37:58.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.222222 9978 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:37:58.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.222481 9978 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:37:58.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.222661 9978 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:37:58.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.222831 9978 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:37:58.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.222977 9978 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40497/jwks.json
05:37:58.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:58.256237 9978 master_runner.cc:386] Master server non-default flags:
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.151.84:42897
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/data
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/wal
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.151.126:43541,127.9.151.125:45533,127.9.151.124:43295
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.151.124:43295
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40497/jwks.json
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/data/info.pb
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.151.124
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:37:58.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:37:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:37:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:37:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/logs
05:37:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:37:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:37:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:37:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:37:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:37:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:37:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:37:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:37:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:37:58.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:37:58.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:58.257550 9978 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:37:58.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:58.259666 9978 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:37:58.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.280820 9989 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:37:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:58.296806 9976 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } attempt: 4
05:37:58.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.303220 9976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.124:43295: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:58.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:58.450060 9901 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } attempt: 7
05:37:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:58.485915 9901 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } has no permanent_uuid. Determining permanent_uuid...
05:37:58.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.495666 9901 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.124:43295: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:58.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:58.554381 9901 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } attempt: 1
05:37:58.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.562868 9901 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.124:43295: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:58.602880 9976 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } attempt: 5
05:37:58.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.609136 9976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.124:43295: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:58.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:58.644439 9901 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } attempt: 2
05:37:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.650913 9901 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.124:43295: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:58.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:58.736464 9901 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } attempt: 3
05:37:58.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.742643 9901 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.124:43295: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:58.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:58.883200 9901 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } attempt: 4
05:37:58.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.890174 9901 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.124:43295: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:59.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.157119 9976 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } attempt: 6
05:37:59.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:59.167937 9976 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.124:43295: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.173727 9901 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } attempt: 5
05:37:59.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:59.180202 9901 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.151.124:43295: Network error: Client connection negotiation failed: client connection to 127.9.151.124:43295: connect: Connection refused (error 111)
05:37:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.280834 9991 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:37:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:58.281390 9988 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:37:59.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:59.543458 9990 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1260 milliseconds
05:37:59.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.543591 9978 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:37:59.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.544828 9978 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:37:59.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.547869 9978 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:37:59.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.549304 9978 hybrid_clock.cc:648] HybridClock initialized: now 1782020279549272 us; error 40 us; skew 500 ppm
05:37:59.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.550076 9978 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:37:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.557824 9978 webserver.cc:460] Webserver started at http://127.9.151.124:38495/ using document root <none> and password file <none>
05:37:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.559130 9978 fs_manager.cc:359] Metadata directory not provided
05:37:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.559630 9978 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:37:59.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.560245 9978 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:37:59.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.565769 9978 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/data/instance:
05:37:59.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4053e74f1ef2416c8464a8a8c1d4de23"
05:37:59.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:37:59 on dist-test-slave-wcx4"
05:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.566857 9978 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/wal/instance:
05:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4053e74f1ef2416c8464a8a8c1d4de23"
05:37:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:37:59 on dist-test-slave-wcx4"
05:37:59.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.574389 9978 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
05:37:59.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.580106 10003 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:37:59.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.581131 9978 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
05:37:59.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.581472 9978 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/data,/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/wal
05:37:59.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4053e74f1ef2416c8464a8a8c1d4de23"
05:37:59.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:37:59 on dist-test-slave-wcx4"
05:37:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.581825 9978 fs_report.cc:383] FS layout report
05:37:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:37:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/wal
05:37:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/wal
05:37:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/data/data
05:37:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:37:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:37:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:37:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:37:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:37:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:37:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:37:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:37:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:37:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:37:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:37:59.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.622696 9978 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:37:59.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.624336 9978 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:37:59.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.624840 9978 kserver.cc:163] Server-wide thread pool size limit: 3276
05:37:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.697708 9978 rpc_server.cc:307] RPC server started. Bound to: 127.9.151.124:43295
05:37:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.697809 10054 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.151.124:43295 every 8 connection(s)
05:37:59.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.700834 9978 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/data/info.pb
05:37:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.707365 10055 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:37:59.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.710695 9821 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu as pid 9978
05:37:59.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.711017 9821 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/master-2/wal/instance
05:37:59.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.722011 10055 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } has no permanent_uuid. Determining permanent_uuid...
05:37:59.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.732681 9901 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } attempt: 6
05:37:59.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.767144 10055 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } has no permanent_uuid. Determining permanent_uuid...
05:37:59.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.775018 9901 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84: Bootstrap starting.
05:37:59.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.781711 10055 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } has no permanent_uuid. Determining permanent_uuid...
05:37:59.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.782342 9901 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84: Neither blocks nor log segments found. Creating new log.
05:37:59.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.784992 9901 log.cc:824] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84: Log is configured to *not* fsync() on all Append() calls
05:37:59.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.789917 9901 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84: No bootstrap required, opened a new log
05:37:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.802117 10055 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23: Bootstrap starting.
05:37:59.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.808409 10055 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23: Neither blocks nor log segments found. Creating new log.
05:37:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.810850 10055 log.cc:824] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23: Log is configured to *not* fsync() on all Append() calls
05:37:59.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.815047 9901 raft_consensus.cc:348] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } }
05:37:59.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.815639 9901 raft_consensus.cc:374] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:37:59.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.816282 10055 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23: No bootstrap required, opened a new log
05:37:59.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.816396 9901 raft_consensus.cc:729] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b7ce5533f44942cd99915c72b0f83f84, State: Initialized, Role: FOLLOWER
05:37:59.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.817230 9901 consensus_queue.cc:260] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } }
05:37:59.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.820451 10067 sys_catalog.cc:455] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } } }
05:37:59.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.821344 10067 sys_catalog.cc:458] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [sys.catalog]: This master's current role is: FOLLOWER
05:37:59.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.822566 9901 sys_catalog.cc:564] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [sys.catalog]: configured and running, proceeding with master startup.
05:37:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.840258 10055 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } }
05:37:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.840909 10055 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:37:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.841163 10055 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4053e74f1ef2416c8464a8a8c1d4de23, State: Initialized, Role: FOLLOWER
05:37:59.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.842067 10055 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } }
05:37:59.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.847488 10055 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [sys.catalog]: configured and running, proceeding with master startup.
05:37:59.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.845722 10077 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } } }
05:37:59.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.857909 10077 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [sys.catalog]: This master's current role is: FOLLOWER
05:37:59.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:59.864172 10079 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:37:59.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:59.864957 10079 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:37:59.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:59.880475 10009 tablet.cc:2369] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:37:59.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.855878 9833 master_runner.cc:186] Error getting master registration for 127.9.151.125:45533: OK, instance_id { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" instance_seqno: 1782020277843189 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:37:59.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.895848 9833 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:37:59.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:59.924769 10091 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:37:59.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:37:59.925107 10091 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:37:59.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.914897 9978 master_runner.cc:186] Error getting master registration for 127.9.151.125:45533: OK, instance_id { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" instance_seqno: 1782020277843189 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:37:59.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.944283 9978 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:37:59.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.956985 10067 raft_consensus.cc:482] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:37:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.957515 10067 raft_consensus.cc:504] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } }
05:37:59.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.960104 10067 leader_election.cc:288] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fc3ccda9f9d34b04ac8780807475e023 (127.9.151.125:45533), 4053e74f1ef2416c8464a8a8c1d4de23 (127.9.151.124:43295)
05:37:59.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.962059 10030 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b7ce5533f44942cd99915c72b0f83f84" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" is_pre_election: true
05:37:59.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.962963 10030 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b7ce5533f44942cd99915c72b0f83f84 in term 0.
05:37:59.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.964184 9852 leader_election.cc:302] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4053e74f1ef2416c8464a8a8c1d4de23, b7ce5533f44942cd99915c72b0f83f84; no voters:
05:37:59.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.965123 10067 raft_consensus.cc:2781] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [term 0 FOLLOWER]: Leader pre-election won for term 1
05:37:59.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.965489 10067 raft_consensus.cc:482] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
05:37:59.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.965803 10067 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [term 0 FOLLOWER]: Advancing to term 1
05:37:59.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.976709 10067 raft_consensus.cc:504] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } }
05:37:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.981678 10067 leader_election.cc:288] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [CANDIDATE]: Term 1 election: Requested vote from peers fc3ccda9f9d34b04ac8780807475e023 (127.9.151.125:45533), 4053e74f1ef2416c8464a8a8c1d4de23 (127.9.151.124:43295)
05:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.982684 10030 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b7ce5533f44942cd99915c72b0f83f84" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4053e74f1ef2416c8464a8a8c1d4de23"
05:37:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.983246 10030 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [term 0 FOLLOWER]: Advancing to term 1
05:37:59.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.990074 10030 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b7ce5533f44942cd99915c72b0f83f84 in term 1.
05:37:59.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.993091 9852 leader_election.cc:302] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4053e74f1ef2416c8464a8a8c1d4de23, b7ce5533f44942cd99915c72b0f83f84; no voters:
05:37:59.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.994443 10067 raft_consensus.cc:2781] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [term 1 FOLLOWER]: Leader election won for term 1
05:37:59.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.997918 10067 raft_consensus.cc:686] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [term 1 LEADER]: Becoming Leader. State: Replica: b7ce5533f44942cd99915c72b0f83f84, State: Running, Role: LEADER
05:38:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:37:59.999280 10067 consensus_queue.cc:237] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } }
05:38:00.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.007491 10093 sys_catalog.cc:455] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b7ce5533f44942cd99915c72b0f83f84. Latest consensus state: current_term: 1 leader_uuid: "b7ce5533f44942cd99915c72b0f83f84" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } } }
05:38:00.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.008368 10093 sys_catalog.cc:458] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [sys.catalog]: This master's current role is: LEADER
05:38:00.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.020423 10095 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
05:38:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.028261 10095 catalog_manager.cc:1484] Initializing Kudu cluster ID...
05:38:00.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:00.035975 9855 tablet.cc:2369] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:38:00.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.051836 10030 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [term 1 FOLLOWER]: Refusing update from remote peer b7ce5533f44942cd99915c72b0f83f84: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:38:00.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.053570 10093 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:38:00.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.084148 10096 mvcc.cc:205] Tried to move back new op lower bound from 7299155067074670592 to 7299155066887872512. Current Snapshot: MvccSnapshot[applied={T|T < 7299155067074670592}]
05:38:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.096493 10067 sys_catalog.cc:455] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b7ce5533f44942cd99915c72b0f83f84" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } } }
05:38:00.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.097295 10067 sys_catalog.cc:458] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [sys.catalog]: This master's current role is: LEADER
05:38:00.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.099712 10095 catalog_manager.cc:1347] Generated new cluster ID: e0dce2921feb4735bc28fec778d79b85
05:38:00.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.100023 10095 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
05:38:00.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.122280 10077 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b7ce5533f44942cd99915c72b0f83f84. Latest consensus state: current_term: 1 leader_uuid: "b7ce5533f44942cd99915c72b0f83f84" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } } }
05:38:00.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.124524 10077 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [sys.catalog]: This master's current role is: FOLLOWER
05:38:00.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.131736 10100 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b7ce5533f44942cd99915c72b0f83f84" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } } }
05:38:00.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.133935 10100 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23 [sys.catalog]: This master's current role is: FOLLOWER
05:38:00.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.146342 10095 catalog_manager.cc:1370] Generated new certificate authority record
05:38:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.149155 10095 catalog_manager.cc:1504] Loading token signing keys...
05:38:00.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.214357 10095 catalog_manager.cc:5868] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84: Generated new TSK 0
05:38:00.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.216032 10095 catalog_manager.cc:1514] Initializing in-progress tserver states...
05:38:00.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.228452 9976 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } attempt: 7
05:38:00.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.256117 9821 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:00.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:00.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/wal
05:38:00.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/data
05:38:00.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:38:00.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/logs
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/data/info.pb
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.151.65:0
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.151.65
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.151.65
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.151.126:43541,127.9.151.125:45533,127.9.151.124:43295
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.151.84:42897
05:38:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:00.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:00.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:38:00.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.273238 9976 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023: Bootstrap starting.
05:38:00.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.285008 9976 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023: Neither blocks nor log segments found. Creating new log.
05:38:00.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.287492 9976 log.cc:824] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023: Log is configured to *not* fsync() on all Append() calls
05:38:00.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.303771 9976 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023: No bootstrap required, opened a new log
05:38:00.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.374591 9976 raft_consensus.cc:348] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } }
05:38:00.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.376889 9976 raft_consensus.cc:374] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:38:00.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.377774 9976 raft_consensus.cc:729] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fc3ccda9f9d34b04ac8780807475e023, State: Initialized, Role: FOLLOWER
05:38:00.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.379925 9976 consensus_queue.cc:260] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } }
05:38:00.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.401211 10106 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } } }
05:38:00.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.403402 10106 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [sys.catalog]: This master's current role is: FOLLOWER
05:38:00.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.408690 9950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b7ce5533f44942cd99915c72b0f83f84" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fc3ccda9f9d34b04ac8780807475e023"
05:38:00.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.409458 9950 raft_consensus.cc:3037] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [term 0 FOLLOWER]: Advancing to term 1
05:38:00.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.413204 9951 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b7ce5533f44942cd99915c72b0f83f84" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fc3ccda9f9d34b04ac8780807475e023" is_pre_election: true
05:38:00.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.419833 9950 raft_consensus.cc:2454] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b7ce5533f44942cd99915c72b0f83f84 in term 1.
05:38:00.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.421507 9949 raft_consensus.cc:1261] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [term 1 FOLLOWER]: Refusing update from remote peer b7ce5533f44942cd99915c72b0f83f84: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:38:00.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.428340 10093 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:38:00.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.440919 9976 sys_catalog.cc:564] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [sys.catalog]: configured and running, proceeding with master startup.
05:38:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.507819 10106 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b7ce5533f44942cd99915c72b0f83f84. Latest consensus state: current_term: 1 leader_uuid: "b7ce5533f44942cd99915c72b0f83f84" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } } }
05:38:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.508606 10106 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [sys.catalog]: This master's current role is: FOLLOWER
05:38:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.508038 10111 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b7ce5533f44942cd99915c72b0f83f84" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } } }
05:38:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.509579 10111 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023 [sys.catalog]: This master's current role is: FOLLOWER
05:38:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.556341 10122 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
05:38:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:00.841632 10104 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:38:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:00.842394 10104 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:38:00.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:00.843171 10104 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:38:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:00.899819 10104 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:38:00.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:00.901408 10104 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.151.65
05:38:00.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.930996 10091 catalog_manager.cc:1259] Loaded cluster ID: e0dce2921feb4735bc28fec778d79b85
05:38:00.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.931598 10091 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23: loading cluster ID for follower catalog manager: success
05:38:00.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.940882 10091 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23: acquiring CA information for follower catalog manager: success
05:38:00.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.946774 10091 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 4053e74f1ef2416c8464a8a8c1d4de23: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:38:00.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.968112 10104 tablet_server_runner.cc:78] Tablet server non-default flags:
05:38:00.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:00.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.151.84:42897
05:38:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/data
05:38:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/wal
05:38:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.151.65:0
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/data/info.pb
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.151.65
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.151.126:43541,127.9.151.125:45533,127.9.151.124:43295
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.151.65
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/logs
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:38:00.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:38:00.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.969647 10104 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:38:00.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:00.971709 10104 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:38:01.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:01.014062 10130 sys_catalog.cc:455] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b7ce5533f44942cd99915c72b0f83f84" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7ce5533f44942cd99915c72b0f83f84" member_type: VOTER last_known_addr { host: "127.9.151.126" port: 43541 } } peers { permanent_uuid: "fc3ccda9f9d34b04ac8780807475e023" member_type: VOTER last_known_addr { host: "127.9.151.125" port: 45533 } } peers { permanent_uuid: "4053e74f1ef2416c8464a8a8c1d4de23" member_type: VOTER last_known_addr { host: "127.9.151.124" port: 43295 } } }
05:38:01.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:01.015169 10130 sys_catalog.cc:458] T 00000000000000000000000000000000 P b7ce5533f44942cd99915c72b0f83f84 [sys.catalog]: This master's current role is: LEADER
05:38:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:01.030340 10128 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:01.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:01.586622 10122 catalog_manager.cc:1259] Loaded cluster ID: e0dce2921feb4735bc28fec778d79b85
05:38:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:01.591990 10122 catalog_manager.cc:1552] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023: loading cluster ID for follower catalog manager: success
05:38:01.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:01.615787 10122 catalog_manager.cc:1574] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023: acquiring CA information for follower catalog manager: success
05:38:01.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:01.629541 10122 catalog_manager.cc:1602] T 00000000000000000000000000000000 P fc3ccda9f9d34b04ac8780807475e023: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:38:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:02.801520 10104 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.770s user 0.426s sys 0.763s
05:38:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:02.802784 10135 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:02.809175 10104 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.778s user 0.427s sys 0.763s
05:38:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:02.808564 10133 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:38:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.809433 10104 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:38:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:02.388563 10127 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 10104
05:38:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:01.032775 10132 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.814978 10104 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:38:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.825618 10104 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:38:02.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.841511 10104 hybrid_clock.cc:648] HybridClock initialized: now 1782020282841451 us; error 67 us; skew 500 ppm
05:38:02.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.842710 10104 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:38:02.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.873397 10104 webserver.cc:460] Webserver started at http://127.9.151.65:45651/ using document root <none> and password file <none>
05:38:02.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.874651 10104 fs_manager.cc:359] Metadata directory not provided
05:38:02.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.874941 10104 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:38:02.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.875478 10104 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:38:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.882505 10104 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/data/instance:
05:38:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eccfbf794c224965b63151dc65dbe595"
05:38:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:02 on dist-test-slave-wcx4"
05:38:02.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.884012 10104 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/wal/instance:
05:38:02.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eccfbf794c224965b63151dc65dbe595"
05:38:02.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:02 on dist-test-slave-wcx4"
05:38:02.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.916200 10104 fs_manager.cc:691] Time spent creating directory manager: real 0.031s user 0.009s sys 0.021s
05:38:02.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.932785 10142 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:38:02.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.940234 10104 fs_manager.cc:725] Time spent opening block manager: real 0.020s user 0.006s sys 0.006s
05:38:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.940662 10104 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/data,/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/wal
05:38:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eccfbf794c224965b63151dc65dbe595"
05:38:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:02 on dist-test-slave-wcx4"
05:38:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:02.941124 10104 fs_report.cc:383] FS layout report
05:38:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/wal
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/wal
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/data/data
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:38:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:38:03.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.008230 10104 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:38:03.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.010143 10104 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:38:03.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.011504 10104 kserver.cc:163] Server-wide thread pool size limit: 3276
05:38:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.020556 10104 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:38:03.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.032649 10104 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:38:03.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.032948 10104 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:38:03.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.033231 10104 ts_tablet_manager.cc:610] Registered 0 tablets
05:38:03.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.033434 10104 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:38:03.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.374595 10104 rpc_server.cc:307] RPC server started. Bound to: 127.9.151.65:34837
05:38:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.374703 10254 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.151.65:34837 every 8 connection(s)
05:38:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.377398 10104 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/data/info.pb
05:38:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.387070 9821 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu as pid 10104
05:38:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.387516 9821 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-0/wal/instance
05:38:03.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.420640 9821 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/wal
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/data
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/logs
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/data/info.pb
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:38:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:38:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:38:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:38:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.151.66:0
05:38:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.151.66
05:38:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.151.66
05:38:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:38:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.151.126:43541,127.9.151.125:45533,127.9.151.124:43295
05:38:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.151.84:42897
05:38:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:03.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:03.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:38:03.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.480332 10255 heartbeater.cc:347] Connected to a master server at 127.9.151.125:45533
05:38:03.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.480762 10255 heartbeater.cc:464] Registering TS with master...
05:38:03.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.481997 10255 heartbeater.cc:511] Master 127.9.151.125:45533 requested a full tablet report, sending...
05:38:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.486289 9941 ts_manager.cc:194] Registered new tserver with Master: eccfbf794c224965b63151dc65dbe595 (127.9.151.65:34837)
05:38:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.486984 10259 heartbeater.cc:347] Connected to a master server at 127.9.151.126:43541
05:38:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.487313 10259 heartbeater.cc:464] Registering TS with master...
05:38:03.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.488164 10259 heartbeater.cc:511] Master 127.9.151.126:43541 requested a full tablet report, sending...
05:38:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.489066 10256 heartbeater.cc:347] Connected to a master server at 127.9.151.124:43295
05:38:03.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.489429 10256 heartbeater.cc:464] Registering TS with master...
05:38:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.490254 10256 heartbeater.cc:511] Master 127.9.151.124:43295 requested a full tablet report, sending...
05:38:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.493259 9866 ts_manager.cc:194] Registered new tserver with Master: eccfbf794c224965b63151dc65dbe595 (127.9.151.65:34837)
05:38:03.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.496634 9866 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.151.65:39403
05:38:03.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.505177 10020 ts_manager.cc:194] Registered new tserver with Master: eccfbf794c224965b63151dc65dbe595 (127.9.151.65:34837)
05:38:03.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:03.914631 10263 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:38:03.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:03.915210 10263 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:38:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:03.915834 10263 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:38:03.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:03.948137 10263 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:38:03.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:03.949182 10263 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.151.66
05:38:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.985839 10263 tablet_server_runner.cc:78] Tablet server non-default flags:
05:38:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.151.84:42897
05:38:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/data
05:38:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/wal
05:38:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.151.66:0
05:38:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:38:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/data/info.pb
05:38:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.151.66
05:38:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:38:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.151.126:43541,127.9.151.125:45533,127.9.151.124:43295
05:38:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:38:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:38:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:38:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.151.66
05:38:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/logs
05:38:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:38:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:38:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:38:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:38:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:38:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:38:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:38:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:38:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:38:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.987423 10263 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:38:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:03.989495 10263 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:38:04.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:04.013731 10271 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:04.502511 10259 heartbeater.cc:503] Master 127.9.151.126:43541 was elected leader, sending a full tablet report...
05:38:05.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:04.015172 10272 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:05.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:04.015388 10274 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:05.362854 10273 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1347 milliseconds
05:38:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.363044 10263 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:38:05.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.364614 10263 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:38:05.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.366870 10263 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:38:05.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.368283 10263 hybrid_clock.cc:648] HybridClock initialized: now 1782020285368251 us; error 57 us; skew 500 ppm
05:38:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.369374 10263 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:38:05.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.377089 10263 webserver.cc:460] Webserver started at http://127.9.151.66:39245/ using document root <none> and password file <none>
05:38:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.378275 10263 fs_manager.cc:359] Metadata directory not provided
05:38:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.378533 10263 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:38:05.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.379070 10263 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:38:05.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.386193 10263 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/data/instance:
05:38:05.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ff188e8170c4961a250b0d6a87c0a1d"
05:38:05.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:05 on dist-test-slave-wcx4"
05:38:05.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.387645 10263 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/wal/instance:
05:38:05.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ff188e8170c4961a250b0d6a87c0a1d"
05:38:05.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:05 on dist-test-slave-wcx4"
05:38:05.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.395009 10263 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.005s
05:38:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.400241 10283 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:38:05.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.401335 10263 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
05:38:05.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.401702 10263 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/data,/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/wal
05:38:05.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ff188e8170c4961a250b0d6a87c0a1d"
05:38:05.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:05 on dist-test-slave-wcx4"
05:38:05.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.402079 10263 fs_report.cc:383] FS layout report
05:38:05.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/wal
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/wal
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/data/data
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:38:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:38:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.468557 10263 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:38:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.470202 10263 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:38:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.470754 10263 kserver.cc:163] Server-wide thread pool size limit: 3276
05:38:05.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.473302 10263 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:38:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.477375 10263 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:38:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.477602 10263 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:38:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.477933 10263 ts_tablet_manager.cc:610] Registered 0 tablets
05:38:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.478122 10263 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:38:05.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.628513 10263 rpc_server.cc:307] RPC server started. Bound to: 127.9.151.66:36911
05:38:05.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.628619 10395 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.151.66:36911 every 8 connection(s)
05:38:05.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.630904 10263 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/data/info.pb
05:38:05.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.633101 9821 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu as pid 10263
05:38:05.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.633737 9821 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-1/wal/instance
05:38:05.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.659260 9821 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/wal
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/data
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/logs
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/data/info.pb
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:05.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:38:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:38:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.151.67:0
05:38:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.151.67
05:38:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.151.67
05:38:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:38:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.151.126:43541,127.9.151.125:45533,127.9.151.124:43295
05:38:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.151.84:42897
05:38:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:38:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.703080 10396 heartbeater.cc:347] Connected to a master server at 127.9.151.125:45533
05:38:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.703459 10398 heartbeater.cc:347] Connected to a master server at 127.9.151.126:43541
05:38:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.703629 10396 heartbeater.cc:464] Registering TS with master...
05:38:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.703812 10398 heartbeater.cc:464] Registering TS with master...
05:38:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.704973 10396 heartbeater.cc:511] Master 127.9.151.125:45533 requested a full tablet report, sending...
05:38:05.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.706379 10398 heartbeater.cc:511] Master 127.9.151.126:43541 requested a full tablet report, sending...
05:38:05.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.707185 10397 heartbeater.cc:347] Connected to a master server at 127.9.151.124:43295
05:38:05.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.707527 10397 heartbeater.cc:464] Registering TS with master...
05:38:05.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.708309 10397 heartbeater.cc:511] Master 127.9.151.124:43295 requested a full tablet report, sending...
05:38:05.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.708554 9941 ts_manager.cc:194] Registered new tserver with Master: 6ff188e8170c4961a250b0d6a87c0a1d (127.9.151.66:36911)
05:38:05.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.708937 9866 ts_manager.cc:194] Registered new tserver with Master: 6ff188e8170c4961a250b0d6a87c0a1d (127.9.151.66:36911)
05:38:05.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.711375 10020 ts_manager.cc:194] Registered new tserver with Master: 6ff188e8170c4961a250b0d6a87c0a1d (127.9.151.66:36911)
05:38:05.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:05.711540 9866 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.151.66:52371
05:38:05.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:05.852720 10051 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6180 after lost signal to thread 9983
05:38:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:06.060570 10406 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:38:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:06.061249 10406 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:38:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:06.061755 10406 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:38:06.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:06.095710 10406 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:38:06.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:06.096647 10406 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.151.67
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:06.149871 10406 tablet_server_runner.cc:78] Tablet server non-default flags:
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.151.84:42897
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/data
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/wal
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.151.67:0
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/data/info.pb
05:38:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.151.67
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.151.126:43541,127.9.151.125:45533,127.9.151.124:43295
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.151.67
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/logs
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:38:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:06.151542 10406 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:38:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:06.154045 10406 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:38:06.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:06.181315 10415 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:06.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:06.715376 10398 heartbeater.cc:503] Master 127.9.151.126:43541 was elected leader, sending a full tablet report...
05:38:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:07.525329 9972 debug-util.cc:398] Leaking SignalData structure 0x7b0800093c60 after lost signal to thread 9907
05:38:07.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:07.526232 9972 debug-util.cc:398] Leaking SignalData structure 0x7b08000ab240 after lost signal to thread 9975
05:38:07.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:06.181320 10413 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:07.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.767722 10406 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:38:07.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:06.185341 10412 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:07.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:07.767596 10414 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1587 milliseconds
05:38:07.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.773852 10406 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:38:07.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.776875 10406 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:38:07.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.778465 10406 hybrid_clock.cc:648] HybridClock initialized: now 1782020287778389 us; error 77 us; skew 500 ppm
05:38:07.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.779760 10406 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:38:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.790057 10406 webserver.cc:460] Webserver started at http://127.9.151.67:33119/ using document root <none> and password file <none>
05:38:07.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.791604 10406 fs_manager.cc:359] Metadata directory not provided
05:38:07.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.792085 10406 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:38:07.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.792805 10406 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:38:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.802263 10406 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/data/instance:
05:38:07.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4835650f56de437b8680bbb1fadca7f9"
05:38:07.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:07 on dist-test-slave-wcx4"
05:38:07.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.804253 10406 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/wal/instance:
05:38:07.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4835650f56de437b8680bbb1fadca7f9"
05:38:07.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:07 on dist-test-slave-wcx4"
05:38:07.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.814924 10406 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.004s sys 0.004s
05:38:07.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.823549 10423 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:38:07.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.825100 10406 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
05:38:07.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.825695 10406 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/data,/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/wal
05:38:07.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4835650f56de437b8680bbb1fadca7f9"
05:38:07.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:07 on dist-test-slave-wcx4"
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.826375 10406 fs_report.cc:383] FS layout report
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/wal
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/wal
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/data/data
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:38:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:38:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.890249 10406 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:38:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.892369 10406 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:38:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.892987 10406 kserver.cc:163] Server-wide thread pool size limit: 3276
05:38:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.897089 10406 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:38:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.904170 10406 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:38:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.904467 10406 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:38:07.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.904803 10406 ts_tablet_manager.cc:610] Registered 0 tablets
05:38:07.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:07.905028 10406 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:38:08.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.188704 10406 rpc_server.cc:307] RPC server started. Bound to: 127.9.151.67:35995
05:38:08.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.188818 10535 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.151.67:35995 every 8 connection(s)
05:38:08.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.192657 10406 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/data/info.pb
05:38:08.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.198292 9821 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu as pid 10406
05:38:08.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.203519 9821 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2178581609775131312/ts-2/wal/instance
05:38:08.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.243100 10537 heartbeater.cc:347] Connected to a master server at 127.9.151.125:45533
05:38:08.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.243595 10537 heartbeater.cc:464] Registering TS with master...
05:38:08.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.244778 10537 heartbeater.cc:511] Master 127.9.151.125:45533 requested a full tablet report, sending...
05:38:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.249645 9941 ts_manager.cc:194] Registered new tserver with Master: 4835650f56de437b8680bbb1fadca7f9 (127.9.151.67:35995)
05:38:08.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.317180 10540 heartbeater.cc:347] Connected to a master server at 127.9.151.124:43295
05:38:08.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.317569 10540 heartbeater.cc:464] Registering TS with master...
05:38:08.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.318451 10540 heartbeater.cc:511] Master 127.9.151.124:43295 requested a full tablet report, sending...
05:38:08.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.323071 10019 ts_manager.cc:194] Registered new tserver with Master: 4835650f56de437b8680bbb1fadca7f9 (127.9.151.67:35995)
05:38:08.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.327718 10541 heartbeater.cc:347] Connected to a master server at 127.9.151.126:43541
05:38:08.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.328122 10541 heartbeater.cc:464] Registering TS with master...
05:38:08.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.328974 10541 heartbeater.cc:511] Master 127.9.151.126:43541 requested a full tablet report, sending...
05:38:08.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.331702 9866 ts_manager.cc:194] Registered new tserver with Master: 4835650f56de437b8680bbb1fadca7f9 (127.9.151.67:35995)
05:38:08.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.333832 9866 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.9.151.67:35255
05:38:08.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:08.344686 9821 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
05:38:08.360 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:38:08.362 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
05:38:08.379 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.9.151.126"
port: 43541
}
bound_http_address {
host: "127.9.151.126"
port: 34611
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.9.151.125"
port: 45533
}
bound_http_address {
host: "127.9.151.125"
port: 41523
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.9.151.124"
port: 43295
}
bound_http_address {
host: "127.9.151.124"
port: 38495
}
}
}
05:38:08.396 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:38:08.408 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.9.151.65"
port: 34837
}
bound_http_address {
host: "127.9.151.65"
port: 45651
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.9.151.66"
port: 36911
}
bound_http_address {
host: "127.9.151.66"
port: 39245
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.9.151.67"
port: 35995
}
bound_http_address {
host: "127.9.151.67"
port: 33119
}
}
}
05:38:08.409 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
05:38:08.690 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
05:38:08.872 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasks6jlpt/test-tmp
05:38:08.873 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:38:08.873 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasks6jlpt/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:38:08.877 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
05:38:09.321 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:38:09.322 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:38:09.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:09.325516 10547 mini_oidc.cc:152] Starting JWKS server
05:38:09.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:09.330336 10547 webserver.cc:460] Webserver started at http://127.0.0.1:42323/ using document root <none> and password file <none>
05:38:09.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:09.330876 10547 mini_oidc.cc:189] Starting OIDC Discovery server
05:38:09.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:09.333541 10547 webserver.cc:460] Webserver started at http://127.0.0.1:35015/ using document root <none> and password file <none>
05:38:09.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:09.339578 10541 heartbeater.cc:503] Master 127.9.151.126:43541 was elected leader, sending a full tablet report...
05:38:09.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
05:38:09.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/principal' for realm 'KRBTEST.COM',
05:38:09.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
05:38:09.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:09 dist-test-slave-wcx4 krb5kdc[10559](info): setting up network...
05:38:09.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
05:38:09.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:09 dist-test-slave-wcx4 krb5kdc[10559](info): set up 2 sockets
05:38:09.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:09 dist-test-slave-wcx4 krb5kdc[10559](info): commencing operation
05:38:09.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
05:38:11.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.002152 10547 mini_kdc.cc:121] Time spent starting KDC: real 1.639s user 0.001s sys 0.020s
05:38:11.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
05:38:11.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:38:11.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
05:38:11.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
05:38:11.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:38:11.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
05:38:11.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
05:38:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:38:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
05:38:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:38:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/test-user.keytab.
05:38:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/test-user.keytab.
05:38:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:11 dist-test-slave-wcx4 krb5kdc[10559](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020291, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:38:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:38:11.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:38:11Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:38:11.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:38:11Z Disabled control of system clock
05:38:11.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.76.254@KRBTEST.COM; defaulting to no policy
05:38:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:38:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.76.254@KRBTEST.COM" created.
05:38:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:38:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.76.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.254.keytab.
05:38:11.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.76.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.254.keytab.
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:11.199050 10547 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/logs
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data/info.pb
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.254:40515
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.254
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.254.keytab
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.76.254
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:38:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.212:36895
05:38:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:38:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.254:40515,127.10.76.253:40359,127.10.76.252:45091
05:38:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:38:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42323/jwks.json
05:38:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:38:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.518682 10573 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:38:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.519385 10573 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:38:11.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.520131 10573 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:38:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.551780 10573 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:38:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.552099 10573 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:38:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.552490 10573 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:38:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.552806 10573 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:38:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.553197 10573 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:38:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.553519 10573 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:38:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.553740 10573 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42323/jwks.json
05:38:11.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:11.600255 10573 master_runner.cc:386] Master server non-default flags:
05:38:11.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:11.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.212:36895
05:38:11.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.254:40515,127.10.76.253:40359,127.10.76.252:45091
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.254.keytab
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.76.254
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.254:40515
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42323/jwks.json
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data/info.pb
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.254
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:38:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:11.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/logs
05:38:11.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:11.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:38:11.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:38:11.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:38:11.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:38:11.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:38:11.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:38:11.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:38:11.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:38:11.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:38:11.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:11.601766 10573 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:38:11.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:11.604053 10573 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:38:11.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.624765 10582 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:12.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.625378 10579 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:12.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:11.627789 10573 server_base.cc:1034] running on GCE node
05:38:12.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:11.624788 10580 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:12.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.857033 10573 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:38:12.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.860481 10573 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:38:12.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.861896 10573 hybrid_clock.cc:648] HybridClock initialized: now 1782020292861830 us; error 78 us; skew 500 ppm
05:38:12.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.862669 10573 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:38:12.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:12 dist-test-slave-wcx4 krb5kdc[10559](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020292, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:38:12.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.868216 10573 init.cc:377] Logged in from keytab as kudu/127.10.76.254@KRBTEST.COM (short username kudu)
05:38:12.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.873296 10573 webserver.cc:460] Webserver started at http://127.10.76.254:40371/ using document root <none> and password file <none>
05:38:12.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.874378 10573 fs_manager.cc:359] Metadata directory not provided
05:38:12.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.874612 10573 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:38:12.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.875125 10573 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:38:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.879758 10573 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data/instance:
05:38:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b605fde513594fe1842c620324efc846"
05:38:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:12 on dist-test-slave-wcx4"
05:38:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.880966 10573 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal/instance:
05:38:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b605fde513594fe1842c620324efc846"
05:38:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:12 on dist-test-slave-wcx4"
05:38:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.888243 10573 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
05:38:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.893622 10590 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:38:12.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.894598 10573 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
05:38:12.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.895051 10573 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data,/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal
05:38:12.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b605fde513594fe1842c620324efc846"
05:38:12.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:12 on dist-test-slave-wcx4"
05:38:12.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.895408 10573 fs_report.cc:383] FS layout report
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data/data
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:38:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:38:12.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.943658 10573 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:38:12.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.949412 10573 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:38:12.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:12.949941 10573 kserver.cc:163] Server-wide thread pool size limit: 3276
05:38:13.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.029762 10573 rpc_server.cc:307] RPC server started. Bound to: 127.10.76.254:40515
05:38:13.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.029881 10641 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.76.254:40515 every 8 connection(s)
05:38:13.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.032437 10573 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data/info.pb
05:38:13.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.037019 10547 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu as pid 10573
05:38:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.037962 10547 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal/instance
05:38:13.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.039393 10642 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:38:13.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.054659 10642 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } has no permanent_uuid. Determining permanent_uuid...
05:38:13.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.76.253@KRBTEST.COM; defaulting to no policy
05:38:13.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:38:13.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.76.253@KRBTEST.COM" created.
05:38:13.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:13 dist-test-slave-wcx4 krb5kdc[10559](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020292, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.254@KRBTEST.COM for kudu/127.10.76.254@KRBTEST.COM
05:38:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:38:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.76.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.253.keytab.
05:38:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.76.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.253.keytab.
05:38:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.089191 10547 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal
05:38:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data
05:38:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:38:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:38:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/logs
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data/info.pb
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.253:40359
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.253
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.253.keytab
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.76.253
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:38:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:38:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.212:36895
05:38:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:38:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.254:40515,127.10.76.253:40359,127.10.76.252:45091
05:38:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:38:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42323/jwks.json
05:38:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:38:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.099099 10642 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } has no permanent_uuid. Determining permanent_uuid...
05:38:13.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.101579 10591 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.106197 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.253:40359: Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:13.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.155658 10642 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } attempt: 1
05:38:13.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.159350 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.253:40359: Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.227738 10642 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } attempt: 2
05:38:13.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.231948 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.253:40359: Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:13.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.323354 10642 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } attempt: 3
05:38:13.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.327589 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.253:40359: Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:13.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.426739 10648 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:38:13.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.427326 10648 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:38:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.427942 10648 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:38:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.459548 10648 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:38:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.459794 10648 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:38:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.460072 10648 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:38:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.460268 10648 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:38:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.460451 10648 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:38:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.460631 10648 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:38:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.460768 10648 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42323/jwks.json
05:38:13.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.470983 10642 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } attempt: 4
05:38:13.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.475221 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.253:40359: Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.498826 10648 master_runner.cc:386] Master server non-default flags:
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.212:36895
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.254:40515,127.10.76.253:40359,127.10.76.252:45091
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.253.keytab
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.76.253
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.253:40359
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:38:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:38:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:38:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42323/jwks.json
05:38:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:38:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:38:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:38:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data/info.pb
05:38:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.253
05:38:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:38:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/logs
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:38:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.500408 10648 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:38:13.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.501824 10648 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:38:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.522023 10655 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:13.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.774605 10642 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } attempt: 5
05:38:13.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.780246 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.253:40359: Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.327824 10642 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } attempt: 6
05:38:14.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:14.340042 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.253:40359: Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.522074 10656 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:13.522064 10658 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:14.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:13.524998 10648 server_base.cc:1034] running on GCE node
05:38:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.768849 10648 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:38:14.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.771534 10648 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:38:14.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.773164 10648 hybrid_clock.cc:648] HybridClock initialized: now 1782020294773092 us; error 83 us; skew 500 ppm
05:38:14.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.774544 10648 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:38:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:14 dist-test-slave-wcx4 krb5kdc[10559](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020294, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:38:14.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.781876 10648 init.cc:377] Logged in from keytab as kudu/127.10.76.253@KRBTEST.COM (short username kudu)
05:38:14.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.787374 10648 webserver.cc:460] Webserver started at http://127.10.76.253:37857/ using document root <none> and password file <none>
05:38:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.788504 10648 fs_manager.cc:359] Metadata directory not provided
05:38:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.788753 10648 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:38:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.789249 10648 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:38:14.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.796460 10648 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data/instance:
05:38:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b005c47e463a416f997eead2edb7b0bd"
05:38:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:14 on dist-test-slave-wcx4"
05:38:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.797829 10648 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal/instance:
05:38:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b005c47e463a416f997eead2edb7b0bd"
05:38:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:14 on dist-test-slave-wcx4"
05:38:14.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.806915 10648 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
05:38:14.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.813719 10671 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:38:14.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.814903 10648 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
05:38:14.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.815275 10648 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data,/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal
05:38:14.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b005c47e463a416f997eead2edb7b0bd"
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:14 on dist-test-slave-wcx4"
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.815604 10648 fs_report.cc:383] FS layout report
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data/data
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:38:14.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:38:14.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.855104 10648 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:38:14.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.861172 10648 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:38:14.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.861687 10648 kserver.cc:163] Server-wide thread pool size limit: 3276
05:38:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.936774 10648 rpc_server.cc:307] RPC server started. Bound to: 127.10.76.253:40359
05:38:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.936853 10722 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.76.253:40359 every 8 connection(s)
05:38:14.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.939474 10648 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data/info.pb
05:38:14.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.944721 10547 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu as pid 10648
05:38:14.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.945217 10547 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal/instance
05:38:14.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.945502 10723 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:38:14.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.956429 10723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } has no permanent_uuid. Determining permanent_uuid...
05:38:14.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.10.76.252@KRBTEST.COM; defaulting to no policy
05:38:14.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:38:14.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.10.76.252@KRBTEST.COM" created.
05:38:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:14 dist-test-slave-wcx4 krb5kdc[10559](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020294, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.253@KRBTEST.COM for kudu/127.10.76.254@KRBTEST.COM
05:38:14.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:38:14.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.76.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.252.keytab.
05:38:14.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.10.76.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.252.keytab.
05:38:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.988133 10547 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/wal
05:38:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data
05:38:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:38:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:38:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:38:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/logs
05:38:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data/info.pb
05:38:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:38:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:38:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:38:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:38:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.252:45091
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.252
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.252.keytab
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.76.252
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.212:36895
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.254:40515,127.10.76.253:40359,127.10.76.252:45091
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42323/jwks.json
05:38:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:38:14.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:14.992689 10723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } has no permanent_uuid. Determining permanent_uuid...
05:38:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:15 dist-test-slave-wcx4 krb5kdc[10559](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020294, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.253@KRBTEST.COM for kudu/127.10.76.253@KRBTEST.COM
05:38:15.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.021816 10723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } has no permanent_uuid. Determining permanent_uuid...
05:38:15.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.024381 10672 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.029186 10723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:15.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.078676 10723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } attempt: 1
05:38:15.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.083971 10723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.152338 10723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } attempt: 2
05:38:15.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.155819 10723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:15.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.247296 10723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } attempt: 3
05:38:15.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.251374 10723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:15.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.313967 10729 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:38:15.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.314692 10729 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:38:15.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.315435 10729 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:38:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.346979 10729 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:38:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.347280 10729 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:38:15.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.347653 10729 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:38:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.347985 10729 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:38:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.348302 10729 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:38:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.348606 10729 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:38:15.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.348829 10729 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42323/jwks.json
05:38:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.393589 10729 master_runner.cc:386] Master server non-default flags:
05:38:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.212:36895
05:38:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data
05:38:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/wal
05:38:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:38:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:38:15.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.254:40515,127.10.76.253:40359,127.10.76.252:45091
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.252.keytab
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.76.252
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.252:45091
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42323/jwks.json
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data/info.pb
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.252
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/logs
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:38:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:38:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:38:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:38:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:38:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:38:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:38:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.394761 10723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } attempt: 4
05:38:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.395078 10729 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:38:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.396479 10729 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:38:15.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.398662 10723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:15.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.400671 10642 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } attempt: 7
05:38:15.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:15 dist-test-slave-wcx4 krb5kdc[10559](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020292, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.254@KRBTEST.COM for kudu/127.10.76.253@KRBTEST.COM
05:38:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.419271 10741 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:15.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.433887 10642 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } has no permanent_uuid. Determining permanent_uuid...
05:38:15.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.439692 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.498128 10642 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } attempt: 1
05:38:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.501791 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:15.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.583398 10642 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } attempt: 2
05:38:15.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.587931 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:15.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.673365 10642 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } attempt: 3
05:38:15.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.678272 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:15.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.698035 10723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } attempt: 5
05:38:15.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.702546 10723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.818706 10642 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } attempt: 4
05:38:15.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.823683 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.107133 10642 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } attempt: 5
05:38:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:16.112242 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:16.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.249931 10723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } attempt: 6
05:38:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:16.257556 10723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.419657 10738 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:16.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:15.423852 10729 server_base.cc:1034] running on GCE node
05:38:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:15.427560 10739 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:16.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.663506 10729 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:38:16.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.664814 10642 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } attempt: 6
05:38:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.667004 10729 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:38:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.668686 10729 hybrid_clock.cc:648] HybridClock initialized: now 1782020296668637 us; error 51 us; skew 500 ppm
05:38:16.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.669764 10729 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:38:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:16.672456 10642 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.10.76.252:45091: Network error: Client connection negotiation failed: client connection to 127.10.76.252:45091: connect: Connection refused (error 111)
05:38:16.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:16 dist-test-slave-wcx4 krb5kdc[10559](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020296, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:38:16.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.677353 10729 init.cc:377] Logged in from keytab as kudu/127.10.76.252@KRBTEST.COM (short username kudu)
05:38:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.683339 10729 webserver.cc:460] Webserver started at http://127.10.76.252:39029/ using document root <none> and password file <none>
05:38:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.684554 10729 fs_manager.cc:359] Metadata directory not provided
05:38:16.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.684815 10729 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:38:16.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.685336 10729 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:38:16.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.692807 10729 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data/instance:
05:38:16.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "069255bfec0349f8b56cb3c468c0cdc6"
05:38:16.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:16 on dist-test-slave-wcx4"
05:38:16.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.694308 10729 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/wal/instance:
05:38:16.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "069255bfec0349f8b56cb3c468c0cdc6"
05:38:16.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:16 on dist-test-slave-wcx4"
05:38:16.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.703979 10729 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.005s
05:38:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.711097 10756 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:38:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.712270 10729 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
05:38:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.712553 10729 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data,/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/wal
05:38:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "069255bfec0349f8b56cb3c468c0cdc6"
05:38:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:16 on dist-test-slave-wcx4"
05:38:16.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.712858 10729 fs_report.cc:383] FS layout report
05:38:16.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:38:16.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/wal
05:38:16.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/wal
05:38:16.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data/data
05:38:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:38:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:38:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:38:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:38:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:38:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:38:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:38:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:38:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:38:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:38:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:38:16.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.751502 10729 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:38:16.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.757210 10729 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:38:16.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.757781 10729 kserver.cc:163] Server-wide thread pool size limit: 3276
05:38:16.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.845099 10729 rpc_server.cc:307] RPC server started. Bound to: 127.10.76.252:45091
05:38:16.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.845131 10807 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.76.252:45091 every 8 connection(s)
05:38:16.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.847839 10729 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data/info.pb
05:38:16.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.854928 10808 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:38:16.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.858158 10547 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu as pid 10729
05:38:16.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.858613 10547 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/wal/instance
05:38:16.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.868868 10808 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } has no permanent_uuid. Determining permanent_uuid...
05:38:16.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:16 dist-test-slave-wcx4 krb5kdc[10559](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020296, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.252@KRBTEST.COM for kudu/127.10.76.254@KRBTEST.COM
05:38:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:16 dist-test-slave-wcx4 krb5kdc[10559](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020291, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.76.252@KRBTEST.COM
05:38:16.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.907358 10808 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } has no permanent_uuid. Determining permanent_uuid...
05:38:16.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:16 dist-test-slave-wcx4 krb5kdc[10559](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020296, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.252@KRBTEST.COM for kudu/127.10.76.253@KRBTEST.COM
05:38:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:16 dist-test-slave-wcx4 krb5kdc[10559](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020291, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.76.253@KRBTEST.COM
05:38:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.935725 10808 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } has no permanent_uuid. Determining permanent_uuid...
05:38:16.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:16 dist-test-slave-wcx4 krb5kdc[10559](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020296, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.252@KRBTEST.COM for kudu/127.10.76.252@KRBTEST.COM
05:38:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:16 dist-test-slave-wcx4 krb5kdc[10559](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020291, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.76.254@KRBTEST.COM
05:38:16.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.977176 10808 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: Bootstrap starting.
05:38:16.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.977609 10547 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
05:38:16.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.984856 10808 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: Neither blocks nor log segments found. Creating new log.
05:38:16.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.986999 10808 log.cc:824] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: Log is configured to *not* fsync() on all Append() calls
05:38:16.988 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:38:16.989 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
05:38:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:16.994234 10808 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: No bootstrap required, opened a new log
05:38:17.001 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
05:38:17.003 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
05:38:17.006 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.76.254"
port: 40515
}
bound_http_address {
host: "127.10.76.254"
port: 40371
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.76.253"
port: 40359
}
bound_http_address {
host: "127.10.76.253"
port: 37857
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.76.252"
port: 45091
}
bound_http_address {
host: "127.10.76.252"
port: 39029
}
}
}
05:38:17.006 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:38:17.007 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:38:17.007 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:38:17.008 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:38:17.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.019550 10808 raft_consensus.cc:348] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b605fde513594fe1842c620324efc846" member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } } peers { permanent_uuid: "b005c47e463a416f997eead2edb7b0bd" member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } } peers { permanent_uuid: "069255bfec0349f8b56cb3c468c0cdc6" member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } }
05:38:17.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.020522 10808 raft_consensus.cc:374] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:38:17.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.020844 10808 raft_consensus.cc:729] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 069255bfec0349f8b56cb3c468c0cdc6, State: Initialized, Role: FOLLOWER
05:38:17.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.021844 10808 consensus_queue.cc:260] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b605fde513594fe1842c620324efc846" member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } } peers { permanent_uuid: "b005c47e463a416f997eead2edb7b0bd" member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } } peers { permanent_uuid: "069255bfec0349f8b56cb3c468c0cdc6" member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } }
05:38:17.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.024897 10818 sys_catalog.cc:455] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b605fde513594fe1842c620324efc846" member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } } peers { permanent_uuid: "b005c47e463a416f997eead2edb7b0bd" member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } } peers { permanent_uuid: "069255bfec0349f8b56cb3c468c0cdc6" member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } } }
05:38:17.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.025962 10818 sys_catalog.cc:458] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [sys.catalog]: This master's current role is: FOLLOWER
05:38:17.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.026571 10808 sys_catalog.cc:564] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [sys.catalog]: configured and running, proceeding with master startup.
05:38:17.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:17 dist-test-slave-wcx4 krb5kdc[10559](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020297, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:38:17.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:38:17.054 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:38:17.057 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5cc
05:38:17.068 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:38:17.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.068734 10830 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:17.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.069092 10830 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:17.069 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5cc
05:38:17.072 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:38:17.077 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
05:38:17.081 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:38:17.082 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:38:17.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.061748 10729 master_runner.cc:186] Error getting master registration for 127.10.76.254:40515: OK, instance_id { permanent_uuid: "b605fde513594fe1842c620324efc846" instance_seqno: 1782020293006648 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:38:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.097790 10729 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:38:17.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:17 dist-test-slave-wcx4 krb5kdc[10559](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020297, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:38:17.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:38:17.125 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:38:17.125 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5cc
05:38:17.127 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:38:17.127 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.76.253:40359
05:38:17.130 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
05:38:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.130945 10547 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu with pid 10648
05:38:17.152 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:38:17.152 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.76.252:45091
05:38:17.153 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
05:38:17.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.153600 10547 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu with pid 10729
05:38:17.176 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:38:17.176 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.10.76.254:40515
05:38:17.177 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
05:38:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.177533 10547 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu with pid 10573
05:38:17.203 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:38:17.204 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.76.253:40359
05:38:17.205 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.207459 10547 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/logs
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data/info.pb
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.253:40359
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.253
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37857
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.253.keytab
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.76.253
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.212:36895
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.254:40515,127.10.76.253:40359,127.10.76.252:45091
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42323/jwks.json
05:38:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:38:17.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.521476 10834 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:38:17.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.522152 10834 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:38:17.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.522884 10834 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:38:17.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.554001 10834 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:38:17.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.554289 10834 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:38:17.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.554751 10834 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:38:17.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.555080 10834 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:38:17.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.555405 10834 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:38:17.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.555729 10834 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:38:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.555975 10834 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42323/jwks.json
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.594869 10834 master_runner.cc:386] Master server non-default flags:
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.212:36895
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.254:40515,127.10.76.253:40359,127.10.76.252:45091
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.253.keytab
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.76.253
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.253:40359
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42323/jwks.json
05:38:17.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data/info.pb
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.253
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37857
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/logs
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:38:17.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.596408 10834 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:38:17.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.597378 10834 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:38:17.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.615978 10840 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.616012 10841 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:18.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:17.616806 10843 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:18.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:17.618453 10834 server_base.cc:1034] running on GCE node
05:38:18.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.844314 10834 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:38:18.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.846866 10834 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:38:18.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.848234 10834 hybrid_clock.cc:648] HybridClock initialized: now 1782020298848186 us; error 58 us; skew 500 ppm
05:38:18.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.849160 10834 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:38:18.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:18 dist-test-slave-wcx4 krb5kdc[10559](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020298, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:38:18.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.860071 10834 init.cc:377] Logged in from keytab as kudu/127.10.76.253@KRBTEST.COM (short username kudu)
05:38:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.865317 10834 webserver.cc:460] Webserver started at http://127.10.76.253:37857/ using document root <none> and password file <none>
05:38:18.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.866362 10834 fs_manager.cc:359] Metadata directory not provided
05:38:18.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.866616 10834 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:38:18.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.875710 10834 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.002s sys 0.005s
05:38:18.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.880086 10852 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:38:18.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.881095 10834 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.003s
05:38:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.881453 10834 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data,/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal
05:38:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b005c47e463a416f997eead2edb7b0bd"
05:38:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:14 on dist-test-slave-wcx4"
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.883682 10834 fs_report.cc:383] FS layout report
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data/data
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:38:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:38:18.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.920753 10834 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:38:18.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.926590 10834 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:38:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.927129 10834 kserver.cc:163] Server-wide thread pool size limit: 3276
05:38:18.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.997394 10834 rpc_server.cc:307] RPC server started. Bound to: 127.10.76.253:40359
05:38:18.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:18.997519 10903 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.76.253:40359 every 8 connection(s)
05:38:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:19.000506 10834 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/data/info.pb
05:38:19.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:19.003799 10547 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu as pid 10834
05:38:19.005 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:38:19.005 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.76.252:45091
05:38:19.005 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
05:38:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:19.007126 10547 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/wal
05:38:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data
05:38:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:38:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:38:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:38:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/logs
05:38:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data/info.pb
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.252:45091
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.252
05:38:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39029
05:38:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.252.keytab
05:38:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.76.252
05:38:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:38:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:38:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:38:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.212:36895
05:38:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:38:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.254:40515,127.10.76.253:40359,127.10.76.252:45091
05:38:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:38:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42323/jwks.json
05:38:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:38:19.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:19.013128 10904 sys_catalog.cc:263] Verifying existing consensus state
05:38:19.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260621 05:38:19.014094 10904 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:38:19.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:19.014601 10834 master.cc:537] Master@127.10.76.253:40359 shutting down...
05:38:19.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:19.076716 10834 master.cc:552] Master@127.10.76.253:40359 shutdown complete.
05:38:19.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:38:19.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:19.345149 10906 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:38:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:19.345842 10906 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:38:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:19.346556 10906 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:38:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:19.381690 10906 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:38:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:19.382014 10906 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:38:19.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:19.382448 10906 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:38:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:19.382790 10906 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:38:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:19.383093 10906 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:38:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:19.383400 10906 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:38:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:19.383617 10906 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42323/jwks.json
05:38:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:19.422236 10906 master_runner.cc:386] Master server non-default flags:
05:38:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.212:36895
05:38:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/wal
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.254:40515,127.10.76.253:40359,127.10.76.252:45091
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.252.keytab
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.76.252
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.252:45091
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42323/jwks.json
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data/info.pb
05:38:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.252
05:38:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39029
05:38:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:38:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:38:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:38:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/logs
05:38:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:38:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:38:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:38:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:38:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:38:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:38:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:38:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:38:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:38:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:19.423460 10906 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:38:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:19.424686 10906 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:38:19.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:19.441715 10914 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:19.441728 10916 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:20.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:19.443011 10913 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:20.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:19.443106 10906 server_base.cc:1034] running on GCE node
05:38:20.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.691555 10906 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:38:20.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.694248 10906 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:38:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.695654 10906 hybrid_clock.cc:648] HybridClock initialized: now 1782020300695620 us; error 69 us; skew 500 ppm
05:38:20.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.696664 10906 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:38:20.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:20 dist-test-slave-wcx4 krb5kdc[10559](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020300, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:38:20.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.703876 10906 init.cc:377] Logged in from keytab as kudu/127.10.76.252@KRBTEST.COM (short username kudu)
05:38:20.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.709748 10906 webserver.cc:460] Webserver started at http://127.10.76.252:39029/ using document root <none> and password file <none>
05:38:20.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.710739 10906 fs_manager.cc:359] Metadata directory not provided
05:38:20.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.710959 10906 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:38:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.719578 10906 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
05:38:20.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.724167 10924 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:38:20.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.725348 10906 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
05:38:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.725637 10906 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data,/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/wal
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "069255bfec0349f8b56cb3c468c0cdc6"
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:16 on dist-test-slave-wcx4"
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.728013 10906 fs_report.cc:383] FS layout report
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/wal
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/wal
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data/data
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:38:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:38:20.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:38:20.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.763249 10906 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:38:20.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.770733 10906 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:38:20.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.771150 10906 kserver.cc:163] Server-wide thread pool size limit: 3276
05:38:20.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.856699 10906 rpc_server.cc:307] RPC server started. Bound to: 127.10.76.252:45091
05:38:20.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.856834 10976 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.76.252:45091 every 8 connection(s)
05:38:20.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.859618 10906 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-2/data/info.pb
05:38:20.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.866060 10547 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu as pid 10906
05:38:20.867 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:38:20.867 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.10.76.254:40515
05:38:20.868 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
05:38:20.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.869534 10547 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:20.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu
05:38:20.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/logs
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data/info.pb
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.254:40515
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.254
05:38:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40371
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.254.keytab
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.76.254
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.212:36895
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.254:40515,127.10.76.253:40359,127.10.76.252:45091
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42323/jwks.json
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:38:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.870977 10977 sys_catalog.cc:263] Verifying existing consensus state
05:38:20.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.876739 10977 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: Bootstrap starting.
05:38:20.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.921759 10977 log.cc:824] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: Log is configured to *not* fsync() on all Append() calls
05:38:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.926702 10977 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
05:38:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.927537 10977 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: Bootstrap complete.
05:38:20.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.961477 10977 raft_consensus.cc:348] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b605fde513594fe1842c620324efc846" member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } } peers { permanent_uuid: "b005c47e463a416f997eead2edb7b0bd" member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } } peers { permanent_uuid: "069255bfec0349f8b56cb3c468c0cdc6" member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } }
05:38:20.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.963955 10977 raft_consensus.cc:374] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:38:20.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.964232 10977 raft_consensus.cc:729] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 069255bfec0349f8b56cb3c468c0cdc6, State: Initialized, Role: FOLLOWER
05:38:20.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.965243 10977 consensus_queue.cc:260] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b605fde513594fe1842c620324efc846" member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } } peers { permanent_uuid: "b005c47e463a416f997eead2edb7b0bd" member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } } peers { permanent_uuid: "069255bfec0349f8b56cb3c468c0cdc6" member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } }
05:38:20.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.968065 10981 sys_catalog.cc:455] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b605fde513594fe1842c620324efc846" member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } } peers { permanent_uuid: "b005c47e463a416f997eead2edb7b0bd" member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } } peers { permanent_uuid: "069255bfec0349f8b56cb3c468c0cdc6" member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } } }
05:38:20.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.968912 10981 sys_catalog.cc:458] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [sys.catalog]: This master's current role is: FOLLOWER
05:38:20.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.969584 10977 sys_catalog.cc:564] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [sys.catalog]: configured and running, proceeding with master startup.
05:38:20.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:20.997829 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:20.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:20.998193 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:21.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:20.995431 10906 master_runner.cc:186] Error getting master registration for 127.10.76.254:40515: Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111),
05:38:21.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:21.016340 10906 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:38:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.026199 10930 tablet.cc:2369] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:38:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:21.046698 10981 raft_consensus.cc:482] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:38:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:21.047228 10981 raft_consensus.cc:504] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b605fde513594fe1842c620324efc846" member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } } peers { permanent_uuid: "b005c47e463a416f997eead2edb7b0bd" member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } } peers { permanent_uuid: "069255bfec0349f8b56cb3c468c0cdc6" member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } }
05:38:21.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:21.052179 10981 leader_election.cc:288] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b605fde513594fe1842c620324efc846 (127.10.76.254:40515), b005c47e463a416f997eead2edb7b0bd (127.10.76.253:40359)
05:38:21.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.052664 10926 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111)
05:38:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.066960 10925 leader_election.cc:334] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b005c47e463a416f997eead2edb7b0bd (127.10.76.253:40359): Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.068490 10926 leader_election.cc:334] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b605fde513594fe1842c620324efc846 (127.10.76.254:40515): Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111)
05:38:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:21.068921 10926 leader_election.cc:302] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 069255bfec0349f8b56cb3c468c0cdc6; no voters: b005c47e463a416f997eead2edb7b0bd, b605fde513594fe1842c620324efc846
05:38:21.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:21.069729 10981 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:38:21.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.251355 10979 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:38:21.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.251936 10979 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:38:21.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.252568 10979 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:38:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.284425 10979 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:38:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.284664 10979 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:38:21.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.284919 10979 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:38:21.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.285118 10979 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:38:21.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.285303 10979 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:38:21.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.285485 10979 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:38:21.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.285627 10979 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42323/jwks.json
05:38:21.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:21.324321 10979 master_runner.cc:386] Master server non-default flags:
05:38:21.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:38:21.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.76.212:36895
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.76.254:40515,127.10.76.253:40359,127.10.76.252:45091
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/krb5kdc/kudu_127.10.76.254.keytab
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.10.76.254
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.76.254:40515
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42323/jwks.json
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data/info.pb
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.76.254
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40371
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:38:21.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:38:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:38:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/logs
05:38:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:38:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:38:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:38:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:38:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:38:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:38:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:38:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:38:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:38:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:38:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:21.325815 10979 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:38:21.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:21.327133 10979 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:38:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.346668 11003 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:22.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:22.004843 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:22.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:22.005307 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:22.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.349037 11005 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:22.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:22.878731 11004 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1534 milliseconds
05:38:22.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.880123 10979 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:38:22.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:21.346681 11002 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:38:22.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.885843 10979 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:38:22.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.889214 10979 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:38:22.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.891475 10979 hybrid_clock.cc:648] HybridClock initialized: now 1782020302891406 us; error 81 us; skew 500 ppm
05:38:22.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.892660 10979 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:38:22.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:22 dist-test-slave-wcx4 krb5kdc[10559](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020302, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:38:22.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.900441 10979 init.cc:377] Logged in from keytab as kudu/127.10.76.254@KRBTEST.COM (short username kudu)
05:38:22.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.906376 10979 webserver.cc:460] Webserver started at http://127.10.76.254:40371/ using document root <none> and password file <none>
05:38:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.907557 10979 fs_manager.cc:359] Metadata directory not provided
05:38:22.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.907845 10979 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:38:22.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.919325 10979 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.006s sys 0.000s
05:38:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.925100 11015 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:38:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.926442 10979 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
05:38:22.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.926846 10979 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data,/tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal
05:38:22.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b605fde513594fe1842c620324efc846"
05:38:22.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:38:12 on dist-test-slave-wcx4"
05:38:22.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.930255 10979 fs_report.cc:383] FS layout report
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data/data
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:38:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:38:22.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.964963 11018 raft_consensus.cc:482] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:38:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.965588 11018 raft_consensus.cc:504] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b605fde513594fe1842c620324efc846" member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } } peers { permanent_uuid: "b005c47e463a416f997eead2edb7b0bd" member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } } peers { permanent_uuid: "069255bfec0349f8b56cb3c468c0cdc6" member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } }
05:38:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.968647 11018 leader_election.cc:288] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b605fde513594fe1842c620324efc846 (127.10.76.254:40515), b005c47e463a416f997eead2edb7b0bd (127.10.76.253:40359)
05:38:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:22.987026 10926 leader_election.cc:334] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b605fde513594fe1842c620324efc846 (127.10.76.254:40515): Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111)
05:38:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:22.987754 10925 leader_election.cc:334] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b005c47e463a416f997eead2edb7b0bd (127.10.76.253:40359): Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.988194 10925 leader_election.cc:302] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 069255bfec0349f8b56cb3c468c0cdc6; no voters: b005c47e463a416f997eead2edb7b0bd, b605fde513594fe1842c620324efc846
05:38:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.989117 11018 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:38:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:22.995918 10979 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:38:23.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:23.004007 10979 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:38:23.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:23.004577 10979 kserver.cc:163] Server-wide thread pool size limit: 3276
05:38:23.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:23.008975 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:23.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:23.009325 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:23.094316 10979 rpc_server.cc:307] RPC server started. Bound to: 127.10.76.254:40515
05:38:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:23.094446 11070 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.76.254:40515 every 8 connection(s)
05:38:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:23.098096 10979 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/data/info.pb
05:38:23.106 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:38:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:23.105203 10547 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu as pid 10979
05:38:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:23.112833 11071 sys_catalog.cc:263] Verifying existing consensus state
05:38:23.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260621 05:38:23.113794 11071 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:38:23.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:23.131011 10979 master.cc:537] Master@127.10.76.254:40515 shutting down...
05:38:23.193 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:23.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:23.235364 10979 master.cc:552] Master@127.10.76.254:40515 shutdown complete.
05:38:23.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasks6jlpt/test-tmp/mini-kudu-cluster2187574918040636952/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:38:23.263 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.76.252:45091(127.10.76.252:45091)] connecting to peer
05:38:23.264 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:23.323 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.10.76.252:45091(127.10.76.252:45091)] Successfully connected to peer
05:38:23.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.326 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.328 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.330 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.331 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.410 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
05:38:23.418 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.10.76.252
05:38:23.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:23 dist-test-slave-wcx4 krb5kdc[10559](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020297, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.10.76.252@KRBTEST.COM
05:38:23.659 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x2c13fa9d, L:/127.0.0.1:39574 - R:/127.10.76.252:45091] using SASL/GSSAPI
05:38:23.679 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:23.699 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:23.704 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:23.708 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.708 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.714 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:23.738 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:23.739 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:23.741 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.742 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.742 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.742 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.743 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.746 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:23.758 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:23.759 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:23.761 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.761 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.761 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.764 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.764 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.765 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.767 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:23.778 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:23.779 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:23.782 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.782 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.783 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.783 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.784 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.784 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.787 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:23.798 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:23.800 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:23.806 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.806 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.807 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.813 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.813 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:23.838 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:23.840 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:23.842 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.842 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.844 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.846 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:23.918 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:23.920 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:23.922 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.923 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.923 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.923 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.924 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:23.924 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:23.926 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:24.012326 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:24.012599 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:24.078 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:24.080 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:24.081 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:24.081 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:24.083 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:24.083 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:24.084 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:24.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:24.087 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:24.338 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:24.339 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:24.341 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:24.341 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:24.341 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:24.341 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:24.342 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:24.342 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:24.344 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:24.398 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:24.399 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:24.402 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:24.402 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:24.402 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:24.403 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:24.404 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:24.404 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:24.406 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:24.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:24.832608 11080 raft_consensus.cc:482] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:38:24.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:24.832966 11080 raft_consensus.cc:504] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b605fde513594fe1842c620324efc846" member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } } peers { permanent_uuid: "b005c47e463a416f997eead2edb7b0bd" member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } } peers { permanent_uuid: "069255bfec0349f8b56cb3c468c0cdc6" member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } }
05:38:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:24.834709 11080 leader_election.cc:288] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b605fde513594fe1842c620324efc846 (127.10.76.254:40515), b005c47e463a416f997eead2edb7b0bd (127.10.76.253:40359)
05:38:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:24.841825 10925 leader_election.cc:334] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b005c47e463a416f997eead2edb7b0bd (127.10.76.253:40359): Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:24.843474 10926 leader_election.cc:334] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b605fde513594fe1842c620324efc846 (127.10.76.254:40515): Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111)
05:38:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:24.843920 10926 leader_election.cc:302] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 069255bfec0349f8b56cb3c468c0cdc6; no voters: b005c47e463a416f997eead2edb7b0bd, b605fde513594fe1842c620324efc846
05:38:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:24.844602 11080 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:38:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:25.015676 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:25.016008 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:25.098 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:25.099 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:25.101 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:25.101 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:25.101 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:25.102 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:25.102 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:25.103 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:25.104 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:25.578 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:25.580 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:25.581 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:25.582 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:25.582 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:25.583 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:25.583 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:25.583 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:25.584 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:26.019050 10906 master_runner.cc:186] Error getting master registration for 127.10.76.254:40515: Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111),
05:38:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:26.019377 10906 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:38:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:26.019838 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:26.020202 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:27.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:27.023655 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:27.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:27.024008 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:27.173806 11087 raft_consensus.cc:482] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:38:27.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:27.174276 11087 raft_consensus.cc:504] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b605fde513594fe1842c620324efc846" member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } } peers { permanent_uuid: "b005c47e463a416f997eead2edb7b0bd" member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } } peers { permanent_uuid: "069255bfec0349f8b56cb3c468c0cdc6" member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } }
05:38:27.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:27.175776 11087 leader_election.cc:288] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b605fde513594fe1842c620324efc846 (127.10.76.254:40515), b005c47e463a416f997eead2edb7b0bd (127.10.76.253:40359)
05:38:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:27.177425 10926 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111) [suppressed 5 similar messages]
05:38:27.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:27.180162 10926 leader_election.cc:334] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b605fde513594fe1842c620324efc846 (127.10.76.254:40515): Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111)
05:38:27.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:27.180553 10925 leader_election.cc:334] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b005c47e463a416f997eead2edb7b0bd (127.10.76.253:40359): Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:27.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:27.180852 10925 leader_election.cc:302] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 069255bfec0349f8b56cb3c468c0cdc6; no voters: b005c47e463a416f997eead2edb7b0bd, b605fde513594fe1842c620324efc846
05:38:27.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:27.181345 11087 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:38:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:28.027321 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:28.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:28.027597 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:29.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:29.030367 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:29.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:29.030627 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:29.538 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:29.540 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:29.541 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:29.541 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:29.541 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:29.541 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:29.542 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:29.542 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:29.544 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:29.718 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:29.719 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:29.721 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:29.721 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:29.721 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:29.723 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:29.723 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:29.723 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:29.725 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:30.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:30.034053 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:30.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:30.034402 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:30.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:30 dist-test-slave-wcx4 krb5kdc[10559](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020310, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:38:30.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:30.707614 10920 init.cc:295] Successfully reacquired a new kerberos TGT
05:38:31.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:31.022488 10906 master_runner.cc:186] Error getting master registration for 127.10.76.254:40515: Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111),
05:38:31.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:31.022940 10906 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:38:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:31.037285 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:31.037580 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:31.501200 10532 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4960 after lost signal to thread 10407
05:38:31.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:31.502144 10532 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4ea0 after lost signal to thread 10535
05:38:32.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:32.040465 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:32.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:32.040768 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:32.458 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:32.461 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:32.461 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:32.461 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:32.461 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:32.463 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:32.464 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:32.464 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:32.464 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:33.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:33.043784 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:33.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:33.044088 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:33.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:33.421363 11099 raft_consensus.cc:482] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:38:33.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:33.421836 11099 raft_consensus.cc:504] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b605fde513594fe1842c620324efc846" member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } } peers { permanent_uuid: "b005c47e463a416f997eead2edb7b0bd" member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } } peers { permanent_uuid: "069255bfec0349f8b56cb3c468c0cdc6" member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } }
05:38:33.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:33.423664 11099 leader_election.cc:288] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b605fde513594fe1842c620324efc846 (127.10.76.254:40515), b005c47e463a416f997eead2edb7b0bd (127.10.76.253:40359)
05:38:33.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:33.425611 10926 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111) [suppressed 1 similar messages]
05:38:33.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:33.428278 10926 leader_election.cc:334] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b605fde513594fe1842c620324efc846 (127.10.76.254:40515): Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111)
05:38:33.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:33.428865 10925 leader_election.cc:334] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b005c47e463a416f997eead2edb7b0bd (127.10.76.253:40359): Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:33.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:33.429149 10925 leader_election.cc:302] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 069255bfec0349f8b56cb3c468c0cdc6; no voters: b005c47e463a416f997eead2edb7b0bd, b605fde513594fe1842c620324efc846
05:38:33.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:33.429672 11099 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:38:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:34.047122 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:34.047370 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:35.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:35.050266 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:35.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:35.050518 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:36.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:36.025590 10906 master_runner.cc:186] Error getting master registration for 127.10.76.254:40515: Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111),
05:38:36.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:36.025923 10906 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:38:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:36.053385 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:36.053660 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:36.338 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:36.339 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:36.340 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:36.340 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:36.340 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:36.341 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:36.341 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:36.341 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:36.342 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:37.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:37.056550 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:37.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:37.056808 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:37.758 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:37.760 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:37.762 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:37.762 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:37.762 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:37.763 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:37.763 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:37.763 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:37.766 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:38.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:38.059686 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:38.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:38.059953 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:38.282886 10251 debug-util.cc:398] Leaking SignalData structure 0x7b08000afc00 after lost signal to thread 10123
05:38:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:38.283655 10251 debug-util.cc:398] Leaking SignalData structure 0x7b08000af8c0 after lost signal to thread 10254
05:38:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:39.064157 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:39.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:39.064517 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:40.068228 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:40.068542 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:40.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:40.404362 11117 raft_consensus.cc:482] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:38:40.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:40.404834 11117 raft_consensus.cc:504] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b605fde513594fe1842c620324efc846" member_type: VOTER last_known_addr { host: "127.10.76.254" port: 40515 } } peers { permanent_uuid: "b005c47e463a416f997eead2edb7b0bd" member_type: VOTER last_known_addr { host: "127.10.76.253" port: 40359 } } peers { permanent_uuid: "069255bfec0349f8b56cb3c468c0cdc6" member_type: VOTER last_known_addr { host: "127.10.76.252" port: 45091 } }
05:38:40.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:40.406539 11117 leader_election.cc:288] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b605fde513594fe1842c620324efc846 (127.10.76.254:40515), b005c47e463a416f997eead2edb7b0bd (127.10.76.253:40359)
05:38:40.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:40.408913 10926 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111) [suppressed 1 similar messages]
05:38:40.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:40.411669 10925 leader_election.cc:334] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b005c47e463a416f997eead2edb7b0bd (127.10.76.253:40359): Network error: Client connection negotiation failed: client connection to 127.10.76.253:40359: connect: Connection refused (error 111)
05:38:40.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:40.412156 10926 leader_election.cc:334] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b605fde513594fe1842c620324efc846 (127.10.76.254:40515): Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111)
05:38:40.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:40.412465 10926 leader_election.cc:302] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 069255bfec0349f8b56cb3c468c0cdc6; no voters: b005c47e463a416f997eead2edb7b0bd, b605fde513594fe1842c620324efc846
05:38:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:40.412998 11117 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:38:40.678 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:40.680 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:40.683 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:40.683 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:40.683 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:40.684 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:40.684 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:40.684 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:40.685 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:40.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:40 dist-test-slave-wcx4 krb5kdc[10559](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020320, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:38:40.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:40.709839 10920 init.cc:295] Successfully reacquired a new kerberos TGT
05:38:41.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:41.028589 10906 master_runner.cc:186] Error getting master registration for 127.10.76.254:40515: Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111),
05:38:41.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:41.028985 10906 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:38:41.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:41.071743 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:41.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:41.072016 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:42.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:42.074910 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:42.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:42.075176 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:43.077769 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:43.078015 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:44.080773 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:44.081032 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:44.518 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:44.520 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:44.521 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:44.521 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:44.522 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:44.522 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:44.522 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:44.522 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:44.524 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:45.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:45.083696 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:45.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:45.083989 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:46.031512 10906 master_runner.cc:186] Error getting master registration for 127.10.76.254:40515: Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111),
05:38:46.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:46.031845 10906 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:38:46.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:46.086760 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:46.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:46.087033 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:46.338 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:46.339 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:46.340 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:46.340 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:46.340 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:46.340 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:46.340 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:46.340 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:46.343 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:47.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:47.089737 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:47.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:47.090010 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:48.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:48.092980 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:48.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:48.093230 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:49.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:49.095922 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:49.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:49.096172 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:49.938 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:49.939 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:49.940 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:49.941 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:49.941 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:49.941 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:49.942 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:49.942 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:49.944 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:50.099305 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:50.099644 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:50.538 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:50.539 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:50.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:50.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:50.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:50.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:50.540 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:50.540 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:50.542 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:50.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:38:50 dist-test-slave-wcx4 krb5kdc[10559](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782020330, etypes {rep=17 tkt=17 ses=17}, kudu/127.10.76.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:38:50.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:50.712525 10920 init.cc:295] Successfully reacquired a new kerberos TGT
05:38:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:51.034972 10906 master_runner.cc:186] Error getting master registration for 127.10.76.254:40515: Network error: Client connection negotiation failed: client connection to 127.10.76.254:40515: connect: Connection refused (error 111),
05:38:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:51.035413 10906 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:38:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:51.102490 10992 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 069255bfec0349f8b56cb3c468c0cdc6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:38:51.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:51.102756 10992 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:38:51.337 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.253:40359(127.10.76.253:40359)] connecting to peer
05:38:51.338 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.10.76.254:40515(127.10.76.254:40515)] connecting to peer
05:38:51.339 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:51.339 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:51.340 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.10.76.254:40515(127.10.76.254:40515)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:51.340 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.253:40359(127.10.76.253:40359)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:51.340 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.254:40515: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515
05:38:51.340 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.10.76.253:40359: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
05:38:51.342 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515; will retry
05:38:51.347 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28226), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [202ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [206ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [549ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [554ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [583ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [586ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [588ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [590ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [612ms] refreshing cache from master, [612ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [616ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [618ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [620ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [621ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [632ms] refreshing cache from master, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [636ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [639ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [641ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [642ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [652ms] refreshing cache from master, [653ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [658ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [658ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [660ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [672ms] refreshing cache from master, [673ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [682ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [686ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [687ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [688ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [712ms] refreshing cache from master, [713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [718ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [719ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [721ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [792ms] refreshing cache from master, [792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [798ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [799ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [800ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [801ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [952ms] refreshing cache from master, [952ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [957ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [959ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [961ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [962ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1212ms] refreshing cache from master, [1213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1216ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1216ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1218ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1219ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1272ms] refreshing cache from master, [1273ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1273ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1274ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1277ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1279ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1281ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1972ms] refreshing cache from master, [1973ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1974ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1974ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1976ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1979ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [2453ms] refreshing cache from master, [2453ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [2454ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [2454ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [2457ms] too many traces: truncated at 100 traces, deferred=Deferred@459810935(state=PENDING, result=null, callback=(continuation of Deferred@465681841 after retry RPC after error@985750088) -> (continuation of Deferred@58931249 after retry RPC after error@1509069734) -> (continuation of Deferred@1338544412 after retry RPC after error@2137751991) -> (continuation of Deferred@1706261723 after retry RPC after error@680635496) -> (continuation of Deferred@593135639 after retry RPC after error@2085874) -> (continuation of Deferred@2117345934 after retry RPC after error@1541616588) -> (continuation of Deferred@739119767 after retry RPC after error@1383122087) -> (continuation of Deferred@227523637 after retry RPC after error@1336967963) -> (continuation of Deferred@334932688 after retry RPC after error@1198981621) -> (continuation of Deferred@938042170 after retry RPC after error@2081068679) -> (continuation of Deferred@1510740988 after retry RPC after error@621533665) -> (continuation of Deferred@1502056137 after retry RPC after error@1896973776) -> (continuation of Deferred@1262025725 after retry RPC after error@805824426) -> (continuation of Deferred@1427359397 after retry RPC after error@676460242) -> (continuation of Deferred@607782710 after retry RPC after error@730832556) -> (continuation of Deferred@1617056662 after retry RPC after error@1193463342) -> (continuation of Deferred@861237835 after retry RPC after error@1685647475) -> (continuation of Deferred@376758665 after retry RPC after error@1198923479) -> (continuation of Deferred@1596582744 after retry RPC after error@2087809958) -> (continuation of Deferred@1295564040 after retry RPC after error@986677982) -> (continuation of Deferred@715967988 after retry RPC after error@1563916100) -> (continuation of Deferred@1441743127 after retry RPC after error@334557190) -> (continuation of Deferred@1610570977 after retry RPC after error@957652237), errback=(continuation of Deferred@465681841 after retry RPC after error@985750088) -> (continuation of Deferred@58931249 after retry RPC after error@1509069734) -> (continuation of Deferred@1338544412 after retry RPC after error@2137751991) -> (continuation of Deferred@1706261723 after retry RPC after error@680635496) -> (continuation of Deferred@593135639 after retry RPC after error@2085874) -> (continuation of Deferred@2117345934 after retry RPC after error@1541616588) -> (continuation of Deferred@739119767 after retry RPC after error@1383122087) -> (continuation of Deferred@227523637 after retry RPC after error@1336967963) -> (continuation of Deferred@334932688 after retry RPC after error@1198981621) -> (continuation of Deferred@938042170 after retry RPC after error@2081068679) -> (continuation of Deferred@1510740988 after retry RPC after error@621533665) -> (continuation of Deferred@1502056137 after retry RPC after error@1896973776) -> (continuation of Deferred@1262025725 after retry RPC after error@805824426) -> (continuation of Deferred@1427359397 after retry RPC after error@676460242) -> (continuation of Deferred@607782710 after retry RPC after error@730832556) -> (continuation of Deferred@1617056662 after retry RPC after error@1193463342) -> (continuation of Deferred@861237835 after retry RPC after error@1685647475) -> (continuation of Deferred@376758665 after retry RPC after error@1198923479) -> (continuation of Deferred@1596582744 after retry RPC after error@2087809958) -> (continuation of Deferred@1295564040 after retry RPC after error@986677982) -> (continuation of Deferred@715967988 after retry RPC after error@1563916100) -> (continuation of Deferred@1441743127 after retry RPC after error@334557190) -> (continuation of Deferred@1610570977 after retry RPC after error@957652237)))
05:38:51.352 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.76.252:45091(127.10.76.252:45091)] disconnecting while in state READY
05:38:51.353 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.76.254:40515(127.10.76.254:40515)] disconnecting while in state TERMINATED
05:38:51.353 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.10.76.253:40359(127.10.76.253:40359)] disconnecting while in state TERMINATED
05:38:51.358 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:38:51.360 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:38:51.360 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.10.76.252:45091(127.10.76.252:45091)] handling channelInactive
05:38:51.361 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.10.76.252:45091(127.10.76.252:45091)] cleaning up while in state READY due to: connection closed
05:38:51.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:51.370976 10547 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu with pid 10906
05:38:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:51.973153 9897 debug-util.cc:398] Leaking SignalData structure 0x7b08000c5140 after lost signal to thread 9834
05:38:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:38:51.974321 9897 debug-util.cc:398] Leaking SignalData structure 0x7b08000c52a0 after lost signal to thread 9900
05:38:52.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:38:52Z chronyd exiting
05:38:53.603 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:38:53.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:53.604349 9821 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu with pid 10104
05:38:53.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:53.637346 9821 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu with pid 10263
05:38:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:53.664081 9821 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu with pid 10406
05:38:53.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:53.694731 9821 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu with pid 9833
05:38:53.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:53.734661 9821 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu with pid 9903
05:38:53.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:38:53.764894 9821 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasks6jlpt/build/tsan/bin/kudu with pid 9978
05:38:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:38:54Z chronyd exiting
05:38:55.950 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28226), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [202ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [206ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [549ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [554ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [583ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [586ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [588ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [590ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [612ms] refreshing cache from master, [612ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [616ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [618ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [620ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [621ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [632ms] refreshing cache from master, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [636ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [639ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [641ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [642ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [652ms] refreshing cache from master, [653ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [658ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [658ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [660ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [672ms] refreshing cache from master, [673ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [682ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [686ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [687ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [688ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [712ms] refreshing cache from master, [713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [718ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [719ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [721ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [792ms] refreshing cache from master, [792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [798ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [799ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [800ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [801ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [952ms] refreshing cache from master, [952ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [957ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [959ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [961ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [962ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1212ms] refreshing cache from master, [1213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1216ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1216ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1218ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1219ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1272ms] refreshing cache from master, [1273ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1273ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1274ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1277ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1279ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1281ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1972ms] refreshing cache from master, [1973ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1974ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1974ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1976ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1979ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [2453ms] refreshing cache from master, [2453ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [2454ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [2454ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [2457ms] too many traces: truncated at 100 traces, deferred=Deferred@459810935(state=PENDING, result=null, callback=(continuation of Deferred@465681841 after retry RPC after error@985750088) -> (continuation of Deferred@58931249 after retry RPC after error@1509069734) -> (continuation of Deferred@1338544412 after retry RPC after error@2137751991) -> (continuation of Deferred@1706261723 after retry RPC after error@680635496) -> (continuation of Deferred@593135639 after retry RPC after error@2085874) -> (continuation of Deferred@2117345934 after retry RPC after error@1541616588) -> (continuation of Deferred@739119767 after retry RPC after error@1383122087) -> (continuation of Deferred@227523637 after retry RPC after error@1336967963) -> (continuation of Deferred@334932688 after retry RPC after error@1198981621) -> (continuation of Deferred@938042170 after retry RPC after error@2081068679) -> (continuation of Deferred@1510740988 after retry RPC after error@621533665) -> (continuation of Deferred@1502056137 after retry RPC after error@1896973776) -> (continuation of Deferred@1262025725 after retry RPC after error@805824426) -> (continuation of Deferred@1427359397 after retry RPC after error@676460242) -> (continuation of Deferred@607782710 after retry RPC after error@730832556) -> (continuation of Deferred@1617056662 after retry RPC after error@1193463342) -> (continuation of Deferred@861237835 after retry RPC after error@1685647475) -> (continuation of Deferred@376758665 after retry RPC after error@1198923479) -> (continuation of Deferred@1596582744 after retry RPC after error@2087809958) -> (continuation of Deferred@1295564040 after retry RPC after error@986677982) -> (continuation of Deferred@715967988 after retry RPC after error@1563916100) -> (continuation of Deferred@1441743127 after retry RPC after error@334557190) -> (continuation of Deferred@1610570977 after retry RPC after error@957652237), errback=(continuation of Deferred@465681841 after retry RPC after error@985750088) -> (continuation of Deferred@58931249 after retry RPC after error@1509069734) -> (continuation of Deferred@1338544412 after retry RPC after error@2137751991) -> (continuation of Deferred@1706261723 after retry RPC after error@680635496) -> (continuation of Deferred@593135639 after retry RPC after error@2085874) -> (continuation of Deferred@2117345934 after retry RPC after error@1541616588) -> (continuation of Deferred@739119767 after retry RPC after error@1383122087) -> (continuation of Deferred@227523637 after retry RPC after error@1336967963) -> (continuation of Deferred@334932688 after retry RPC after error@1198981621) -> (continuation of Deferred@938042170 after retry RPC after error@2081068679) -> (continuation of Deferred@1510740988 after retry RPC after error@621533665) -> (continuation of Deferred@1502056137 after retry RPC after error@1896973776) -> (continuation of Deferred@1262025725 after retry RPC after error@805824426) -> (continuation of Deferred@1427359397 after retry RPC after error@676460242) -> (continuation of Deferred@607782710 after retry RPC after error@730832556) -> (continuation of Deferred@1617056662 after retry RPC after error@1193463342) -> (continuation of Deferred@861237835 after retry RPC after error@1685647475) -> (continuation of Deferred@376758665 after retry RPC after error@1198923479) -> (continuation of Deferred@1596582744 after retry RPC after error@2087809958) -> (continuation of Deferred@1295564040 after retry RPC after error@986677982) -> (continuation of Deferred@715967988 after retry RPC after error@1563916100) -> (continuation of Deferred@1441743127 after retry RPC after error@334557190) -> (continuation of Deferred@1610570977 after retry RPC after error@957652237)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.76.253:40359
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28226), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [202ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [206ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [549ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [554ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [583ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [586ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [588ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [590ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [612ms] refreshing cache from master, [612ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [616ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [618ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [620ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [621ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [632ms] refreshing cache from master, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [636ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [639ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [641ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [642ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [652ms] refreshing cache from master, [653ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [658ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [658ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [660ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [672ms] refreshing cache from master, [673ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [682ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [686ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [687ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [688ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [712ms] refreshing cache from master, [713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [718ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [719ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [721ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [792ms] refreshing cache from master, [792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [798ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [799ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [800ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [801ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [952ms] refreshing cache from master, [952ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [957ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [959ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [961ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [962ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1212ms] refreshing cache from master, [1213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1216ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1216ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1218ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1219ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1272ms] refreshing cache from master, [1273ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1273ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1274ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1277ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1279ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1281ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1972ms] refreshing cache from master, [1973ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1974ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1974ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1976ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1979ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [2453ms] refreshing cache from master, [2453ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [2454ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [2454ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [2457ms] too many traces: truncated at 100 traces, deferred=Deferred@459810935(state=PENDING, result=null, callback=(continuation of Deferred@465681841 after retry RPC after error@985750088) -> (continuation of Deferred@58931249 after retry RPC after error@1509069734) -> (continuation of Deferred@1338544412 after retry RPC after error@2137751991) -> (continuation of Deferred@1706261723 after retry RPC after error@680635496) -> (continuation of Deferred@593135639 after retry RPC after error@2085874) -> (continuation of Deferred@2117345934 after retry RPC after error@1541616588) -> (continuation of Deferred@739119767 after retry RPC after error@1383122087) -> (continuation of Deferred@227523637 after retry RPC after error@1336967963) -> (continuation of Deferred@334932688 after retry RPC after error@1198981621) -> (continuation of Deferred@938042170 after retry RPC after error@2081068679) -> (continuation of Deferred@1510740988 after retry RPC after error@621533665) -> (continuation of Deferred@1502056137 after retry RPC after error@1896973776) -> (continuation of Deferred@1262025725 after retry RPC after error@805824426) -> (continuation of Deferred@1427359397 after retry RPC after error@676460242) -> (continuation of Deferred@607782710 after retry RPC after error@730832556) -> (continuation of Deferred@1617056662 after retry RPC after error@1193463342) -> (continuation of Deferred@861237835 after retry RPC after error@1685647475) -> (continuation of Deferred@376758665 after retry RPC after error@1198923479) -> (continuation of Deferred@1596582744 after retry RPC after error@2087809958) -> (continuation of Deferred@1295564040 after retry RPC after error@986677982) -> (continuation of Deferred@715967988 after retry RPC after error@1563916100) -> (continuation of Deferred@1441743127 after retry RPC after error@334557190) -> (continuation of Deferred@1610570977 after retry RPC after error@957652237), errback=(continuation of Deferred@465681841 after retry RPC after error@985750088) -> (continuation of Deferred@58931249 after retry RPC after error@1509069734) -> (continuation of Deferred@1338544412 after retry RPC after error@2137751991) -> (continuation of Deferred@1706261723 after retry RPC after error@680635496) -> (continuation of Deferred@593135639 after retry RPC after error@2085874) -> (continuation of Deferred@2117345934 after retry RPC after error@1541616588) -> (continuation of Deferred@739119767 after retry RPC after error@1383122087) -> (continuation of Deferred@227523637 after retry RPC after error@1336967963) -> (continuation of Deferred@334932688 after retry RPC after error@1198981621) -> (continuation of Deferred@938042170 after retry RPC after error@2081068679) -> (continuation of Deferred@1510740988 after retry RPC after error@621533665) -> (continuation of Deferred@1502056137 after retry RPC after error@1896973776) -> (continuation of Deferred@1262025725 after retry RPC after error@805824426) -> (continuation of Deferred@1427359397 after retry RPC after error@676460242) -> (continuation of Deferred@607782710 after retry RPC after error@730832556) -> (continuation of Deferred@1617056662 after retry RPC after error@1193463342) -> (continuation of Deferred@861237835 after retry RPC after error@1685647475) -> (continuation of Deferred@376758665 after retry RPC after error@1198923479) -> (continuation of Deferred@1596582744 after retry RPC after error@2087809958) -> (continuation of Deferred@1295564040 after retry RPC after error@986677982) -> (continuation of Deferred@715967988 after retry RPC after error@1563916100) -> (continuation of Deferred@1441743127 after retry RPC after error@334557190) -> (continuation of Deferred@1610570977 after retry RPC after error@957652237)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.76.253:40359
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
05:38:56.362 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
05:38:56.363 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28226), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [202ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [206ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [549ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [554ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [583ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [586ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [588ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [590ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [612ms] refreshing cache from master, [612ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [616ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [618ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [620ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [621ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [632ms] refreshing cache from master, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [636ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [639ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [641ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [642ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [652ms] refreshing cache from master, [653ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [658ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [658ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [660ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [672ms] refreshing cache from master, [673ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [682ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [686ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [687ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [688ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [712ms] refreshing cache from master, [713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [718ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [719ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [721ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [792ms] refreshing cache from master, [792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [798ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [799ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [800ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [801ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [952ms] refreshing cache from master, [952ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [957ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [959ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [961ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [962ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1212ms] refreshing cache from master, [1213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1216ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1216ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1218ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1219ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1272ms] refreshing cache from master, [1273ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1273ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1274ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1277ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1279ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1281ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1972ms] refreshing cache from master, [1973ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1974ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1974ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1976ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1979ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [2453ms] refreshing cache from master, [2453ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [2454ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [2454ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [2457ms] too many traces: truncated at 100 traces, deferred=Deferred@459810935(state=PENDING, result=null, callback=(continuation of Deferred@465681841 after retry RPC after error@985750088) -> (continuation of Deferred@58931249 after retry RPC after error@1509069734) -> (continuation of Deferred@1338544412 after retry RPC after error@2137751991) -> (continuation of Deferred@1706261723 after retry RPC after error@680635496) -> (continuation of Deferred@593135639 after retry RPC after error@2085874) -> (continuation of Deferred@2117345934 after retry RPC after error@1541616588) -> (continuation of Deferred@739119767 after retry RPC after error@1383122087) -> (continuation of Deferred@227523637 after retry RPC after error@1336967963) -> (continuation of Deferred@334932688 after retry RPC after error@1198981621) -> (continuation of Deferred@938042170 after retry RPC after error@2081068679) -> (continuation of Deferred@1510740988 after retry RPC after error@621533665) -> (continuation of Deferred@1502056137 after retry RPC after error@1896973776) -> (continuation of Deferred@1262025725 after retry RPC after error@805824426) -> (continuation of Deferred@1427359397 after retry RPC after error@676460242) -> (continuation of Deferred@607782710 after retry RPC after error@730832556) -> (continuation of Deferred@1617056662 after retry RPC after error@1193463342) -> (continuation of Deferred@861237835 after retry RPC after error@1685647475) -> (continuation of Deferred@376758665 after retry RPC after error@1198923479) -> (continuation of Deferred@1596582744 after retry RPC after error@2087809958) -> (continuation of Deferred@1295564040 after retry RPC after error@986677982) -> (continuation of Deferred@715967988 after retry RPC after error@1563916100) -> (continuation of Deferred@1441743127 after retry RPC after error@334557190) -> (continuation of Deferred@1610570977 after retry RPC after error@957652237), errback=(continuation of Deferred@465681841 after retry RPC after error@985750088) -> (continuation of Deferred@58931249 after retry RPC after error@1509069734) -> (continuation of Deferred@1338544412 after retry RPC after error@2137751991) -> (continuation of Deferred@1706261723 after retry RPC after error@680635496) -> (continuation of Deferred@593135639 after retry RPC after error@2085874) -> (continuation of Deferred@2117345934 after retry RPC after error@1541616588) -> (continuation of Deferred@739119767 after retry RPC after error@1383122087) -> (continuation of Deferred@227523637 after retry RPC after error@1336967963) -> (continuation of Deferred@334932688 after retry RPC after error@1198981621) -> (continuation of Deferred@938042170 after retry RPC after error@2081068679) -> (continuation of Deferred@1510740988 after retry RPC after error@621533665) -> (continuation of Deferred@1502056137 after retry RPC after error@1896973776) -> (continuation of Deferred@1262025725 after retry RPC after error@805824426) -> (continuation of Deferred@1427359397 after retry RPC after error@676460242) -> (continuation of Deferred@607782710 after retry RPC after error@730832556) -> (continuation of Deferred@1617056662 after retry RPC after error@1193463342) -> (continuation of Deferred@861237835 after retry RPC after error@1685647475) -> (continuation of Deferred@376758665 after retry RPC after error@1198923479) -> (continuation of Deferred@1596582744 after retry RPC after error@2087809958) -> (continuation of Deferred@1295564040 after retry RPC after error@986677982) -> (continuation of Deferred@715967988 after retry RPC after error@1563916100) -> (continuation of Deferred@1441743127 after retry RPC after error@334557190) -> (continuation of Deferred@1610570977 after retry RPC after error@957652237)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.76.253:40359
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28226), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [202ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [206ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [549ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [554ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [573ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [583ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [586ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [588ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [590ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [612ms] refreshing cache from master, [612ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [616ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [618ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [620ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [621ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [632ms] refreshing cache from master, [633ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [634ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [636ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [639ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [641ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [642ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [652ms] refreshing cache from master, [653ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [658ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [658ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [660ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [672ms] refreshing cache from master, [673ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [682ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [686ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [687ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [688ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [712ms] refreshing cache from master, [713ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [714ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [718ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [719ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [720ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [721ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [792ms] refreshing cache from master, [792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [794ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [798ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [799ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [800ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [801ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [952ms] refreshing cache from master, [952ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [954ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [957ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [959ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [961ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [962ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1212ms] refreshing cache from master, [1213ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1216ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1216ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1218ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1219ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1272ms] refreshing cache from master, [1273ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1273ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1274ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1277ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1279ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1281ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1972ms] refreshing cache from master, [1973ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [1974ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [1974ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [1976ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.253:40359: Network error: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.254:40515: Network error: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.10.76.252:45091: OK, [1979ms] delaying RPC due to: Service unavailable: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515, [2453ms] refreshing cache from master, [2453ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.253:40359, [2454ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.252:45091, [2454ms] Sub RPC ConnectToMaster: sending RPC to server master-127.10.76.254:40515, [2457ms] too many traces: truncated at 100 traces, deferred=Deferred@459810935(state=PENDING, result=null, callback=(continuation of Deferred@465681841 after retry RPC after error@985750088) -> (continuation of Deferred@58931249 after retry RPC after error@1509069734) -> (continuation of Deferred@1338544412 after retry RPC after error@2137751991) -> (continuation of Deferred@1706261723 after retry RPC after error@680635496) -> (continuation of Deferred@593135639 after retry RPC after error@2085874) -> (continuation of Deferred@2117345934 after retry RPC after error@1541616588) -> (continuation of Deferred@739119767 after retry RPC after error@1383122087) -> (continuation of Deferred@227523637 after retry RPC after error@1336967963) -> (continuation of Deferred@334932688 after retry RPC after error@1198981621) -> (continuation of Deferred@938042170 after retry RPC after error@2081068679) -> (continuation of Deferred@1510740988 after retry RPC after error@621533665) -> (continuation of Deferred@1502056137 after retry RPC after error@1896973776) -> (continuation of Deferred@1262025725 after retry RPC after error@805824426) -> (continuation of Deferred@1427359397 after retry RPC after error@676460242) -> (continuation of Deferred@607782710 after retry RPC after error@730832556) -> (continuation of Deferred@1617056662 after retry RPC after error@1193463342) -> (continuation of Deferred@861237835 after retry RPC after error@1685647475) -> (continuation of Deferred@376758665 after retry RPC after error@1198923479) -> (continuation of Deferred@1596582744 after retry RPC after error@2087809958) -> (continuation of Deferred@1295564040 after retry RPC after error@986677982) -> (continuation of Deferred@715967988 after retry RPC after error@1563916100) -> (continuation of Deferred@1441743127 after retry RPC after error@334557190) -> (continuation of Deferred@1610570977 after retry RPC after error@957652237), errback=(continuation of Deferred@465681841 after retry RPC after error@985750088) -> (continuation of Deferred@58931249 after retry RPC after error@1509069734) -> (continuation of Deferred@1338544412 after retry RPC after error@2137751991) -> (continuation of Deferred@1706261723 after retry RPC after error@680635496) -> (continuation of Deferred@593135639 after retry RPC after error@2085874) -> (continuation of Deferred@2117345934 after retry RPC after error@1541616588) -> (continuation of Deferred@739119767 after retry RPC after error@1383122087) -> (continuation of Deferred@227523637 after retry RPC after error@1336967963) -> (continuation of Deferred@334932688 after retry RPC after error@1198981621) -> (continuation of Deferred@938042170 after retry RPC after error@2081068679) -> (continuation of Deferred@1510740988 after retry RPC after error@621533665) -> (continuation of Deferred@1502056137 after retry RPC after error@1896973776) -> (continuation of Deferred@1262025725 after retry RPC after error@805824426) -> (continuation of Deferred@1427359397 after retry RPC after error@676460242) -> (continuation of Deferred@607782710 after retry RPC after error@730832556) -> (continuation of Deferred@1617056662 after retry RPC after error@1193463342) -> (continuation of Deferred@861237835 after retry RPC after error@1685647475) -> (continuation of Deferred@376758665 after retry RPC after error@1198923479) -> (continuation of Deferred@1596582744 after retry RPC after error@2087809958) -> (continuation of Deferred@1295564040 after retry RPC after error@986677982) -> (continuation of Deferred@715967988 after retry RPC after error@1563916100) -> (continuation of Deferred@1441743127 after retry RPC after error@334557190) -> (continuation of Deferred@1610570977 after retry RPC after error@957652237)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.10.76.253:40359,127.10.76.252:45091,127.10.76.254:40515) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.254:40515(127.10.76.254:40515): Connection refused: /127.10.76.254:40515,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.10.76.253:40359(127.10.76.253:40359): Connection refused: /127.10.76.253:40359
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.10.76.253:40359
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)