14:02:52.821 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:02:52.821 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task_tJAxU/test-tmp
14:02:52.822 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:02:52.822 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task_tJAxU/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:02:52.824 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:02:53.168 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:02:53.169 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:02:53.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:53.172323 22868 mini_oidc.cc:150] Starting JWKS server
14:02:53.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:53.175664 22868 webserver.cc:533] Webserver started at http://127.0.0.1:44443/ using document root <none> and password file <none>
14:02:53.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:53.176083 22868 mini_oidc.cc:186] Starting OIDC Discovery server
14:02:53.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:53.177831 22868 webserver.cc:533] Webserver started at http://127.0.0.1:38227/ using document root <none> and password file <none>
14:02:53.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T14:02:53Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:02:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T14:02:53Z Disabled control of system clock
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:53.248190 22868 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/wal
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/data
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/logs
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/data/info.pb
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:02:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.85.62:38623
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.85.62
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.85.20:45841
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.85.62:38623,127.22.85.61:41445,127.22.85.60:41663
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:02:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44443/jwks.json with env {}
14:02:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:53.568387 22880 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:02:53.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:53.568984 22880 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:02:53.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:53.569466 22880 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:02:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:53.603755 22880 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:02:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:53.604067 22880 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:02:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:53.604278 22880 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:02:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:53.604501 22880 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:02:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:53.604703 22880 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:02:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:53.604851 22880 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:44443/jwks.json
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:53.643630 22880 master_runner.cc:387] Master server non-default flags:
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/logs
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.85.20:45841
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/data
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/wal
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.85.62:38623,127.22.85.61:41445,127.22.85.60:41663
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.85.62:38623
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44443/jwks.json
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/data/info.pb
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.85.62
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cb747ff0ca0209e3fa98be4167c67edd474acd67
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Aug 2026 13:43:25 UTC on e7f111948823
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13227
14:02:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:02:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:53.645212 22880 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:02:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:53.646916 22880 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:02:53.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:53.659101 22889 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:02:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.062678 22886 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee00 after lost signal to thread 22880
14:02:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.219027 22880 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.560s user 0.546s sys 1.010s
14:02:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.219522 22880 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.561s user 0.546s sys 1.010s
14:02:55.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:53.659106 22888 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:02:55.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.221688 22891 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:02:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.224664 22890 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1555 milliseconds
14:02:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.224695 22880 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:02:55.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.225991 22880 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:02:55.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.229095 22880 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:02:55.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.230549 22880 hybrid_clock.cc:648] HybridClock initialized: now 1786111375230507 us; error 52 us; skew 500 ppm
14:02:55.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.231377 22880 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:02:55.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.238585 22880 webserver.cc:533] Webserver started at http://127.22.85.62:41951/ using document root <none> and password file <none>
14:02:55.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.239923 22880 fs_manager.cc:362] Metadata directory not provided
14:02:55.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.240260 22880 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:02:55.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.240958 22880 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:02:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.248253 22880 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/data/instance:
14:02:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "38b700491d2d4374aaf7ff824ee95a97"
14:02:55.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:02:55 on dist-test-slave-10pc"
14:02:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.250062 22880 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/wal/instance:
14:02:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "38b700491d2d4374aaf7ff824ee95a97"
14:02:55.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:02:55 on dist-test-slave-10pc"
14:02:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.259745 22880 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.005s
14:02:55.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.268306 22898 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:02:55.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.269676 22880 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
14:02:55.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.270041 22880 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/data,/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/wal
14:02:55.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "38b700491d2d4374aaf7ff824ee95a97"
14:02:55.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:02:55 on dist-test-slave-10pc"
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.270435 22880 fs_report.cc:389] FS layout report
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/wal
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/wal
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/data/data
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:02:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:02:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.333899 22880 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:02:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.335584 22880 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:02:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.336110 22880 kserver.cc:163] Server-wide thread pool size limit: 3276
14:02:55.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.414376 22880 rpc_server.cc:307] RPC server started. Bound to: 127.22.85.62:38623
14:02:55.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.414511 22955 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.85.62:38623 every 8 connection(s)
14:02:55.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.417295 22880 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/data/info.pb
14:02:55.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.423907 22956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:02:55.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.425915 22868 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu as pid 22880
14:02:55.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.426396 22868 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-0/wal/instance
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.429828 22868 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/wal
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/data
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/logs
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/data/info.pb
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:02:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.85.61:41445
14:02:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.85.61
14:02:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:02:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.85.20:45841
14:02:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:02:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:02:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:02:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:02:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.85.62:38623,127.22.85.61:41445,127.22.85.60:41663
14:02:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:02:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44443/jwks.json with env {}
14:02:55.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.434962 22956 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } has no permanent_uuid. Determining permanent_uuid...
14:02:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.465652 22956 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } has no permanent_uuid. Determining permanent_uuid...
14:02:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.468312 22899 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.85.61:41445: connect: Connection refused (error 111)
14:02:55.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.473127 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.61:41445: Network error: Client connection negotiation failed: client connection to 127.22.85.61:41445: connect: Connection refused (error 111)
14:02:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.522722 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } attempt: 1
14:02:55.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.527365 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.61:41445: Network error: Client connection negotiation failed: client connection to 127.22.85.61:41445: connect: Connection refused (error 111)
14:02:55.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.595878 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } attempt: 2
14:02:55.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.600525 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.61:41445: Network error: Client connection negotiation failed: client connection to 127.22.85.61:41445: connect: Connection refused (error 111)
14:02:55.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.692039 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } attempt: 3
14:02:55.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.697393 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.61:41445: Network error: Client connection negotiation failed: client connection to 127.22.85.61:41445: connect: Connection refused (error 111)
14:02:55.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.762357 22958 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:02:55.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.763079 22958 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:02:55.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.763705 22958 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:02:55.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.798816 22958 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:02:55.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.799232 22958 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:02:55.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.799588 22958 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:02:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.799975 22958 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:02:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.800294 22958 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:02:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.800558 22958 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:44443/jwks.json
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.839643 22958 master_runner.cc:387] Master server non-default flags:
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/logs
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.85.20:45841
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/data
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/wal
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.85.62:38623,127.22.85.61:41445,127.22.85.60:41663
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.85.61:41445
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44443/jwks.json
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:02:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/data/info.pb
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.85.61
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cb747ff0ca0209e3fa98be4167c67edd474acd67
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Aug 2026 13:43:25 UTC on e7f111948823
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13227
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.840870 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } attempt: 4
14:02:55.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.841475 22958 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:02:55.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:55.843765 22958 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:02:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.845853 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.61:41445: Network error: Client connection negotiation failed: client connection to 127.22.85.61:41445: connect: Connection refused (error 111)
14:02:55.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.855356 22967 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:02:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:56.145345 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } attempt: 5
14:02:56.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:56.154963 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.61:41445: Network error: Client connection negotiation failed: client connection to 127.22.85.61:41445: connect: Connection refused (error 111)
14:02:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:56.702600 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } attempt: 6
14:02:56.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:56.723961 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.61:41445: Network error: Client connection negotiation failed: client connection to 127.22.85.61:41445: connect: Connection refused (error 111)
14:02:57.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:55.857334 22968 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:02:57.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.122745 22969 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1263 milliseconds
14:02:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.124715 22970 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:02:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.128052 22958 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.272s user 0.000s sys 0.008s
14:02:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.128322 22958 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.272s user 0.000s sys 0.008s
14:02:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.128610 22958 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:02:57.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.129835 22958 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:02:57.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.132767 22958 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:02:57.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.134285 22958 hybrid_clock.cc:648] HybridClock initialized: now 1786111377134247 us; error 40 us; skew 500 ppm
14:02:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.135213 22958 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:02:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.143668 22958 webserver.cc:533] Webserver started at http://127.22.85.61:39263/ using document root <none> and password file <none>
14:02:57.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.144768 22958 fs_manager.cc:362] Metadata directory not provided
14:02:57.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.145049 22958 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:02:57.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.145586 22958 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:02:57.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.150133 22958 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/data/instance:
14:02:57.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a976288f24f47099dfaaeaa7210928d"
14:02:57.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:02:57 on dist-test-slave-10pc"
14:02:57.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.151417 22958 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/wal/instance:
14:02:57.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a976288f24f47099dfaaeaa7210928d"
14:02:57.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:02:57 on dist-test-slave-10pc"
14:02:57.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.160002 22958 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
14:02:57.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.167021 22979 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:02:57.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.168387 22958 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.168823 22958 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/data,/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/wal
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a976288f24f47099dfaaeaa7210928d"
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:02:57 on dist-test-slave-10pc"
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.169205 22958 fs_report.cc:389] FS layout report
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/wal
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/wal
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/data/data
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:02:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:02:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:02:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:02:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:02:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:02:57.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:02:57.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.244776 22958 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:02:57.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.246404 22958 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:02:57.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.246930 22958 kserver.cc:163] Server-wide thread pool size limit: 3276
14:02:57.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.321509 22958 rpc_server.cc:307] RPC server started. Bound to: 127.22.85.61:41445
14:02:57.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.321599 23032 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.85.61:41445 every 8 connection(s)
14:02:57.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.324642 22958 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/data/info.pb
14:02:57.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.330585 22868 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu as pid 22958
14:02:57.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.331104 22868 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-1/wal/instance
14:02:57.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.331326 23033 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.334198 22868 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/wal
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/data
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/logs
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/data/info.pb
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.85.60:41663
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.85.60
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:02:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.85.20:45841
14:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.85.62:38623,127.22.85.61:41445,127.22.85.60:41663
14:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:02:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44443/jwks.json with env {}
14:02:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.346727 23033 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } has no permanent_uuid. Determining permanent_uuid...
14:02:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.369412 23033 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } has no permanent_uuid. Determining permanent_uuid...
14:02:57.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.385035 23033 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } has no permanent_uuid. Determining permanent_uuid...
14:02:57.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.387365 22982 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:57.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.391659 23033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.441195 23033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 1
14:02:57.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.445986 23033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:57.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.514420 23033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 2
14:02:57.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.519069 23033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:57.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.610529 23033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 3
14:02:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.615685 23033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:57.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.674073 23035 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:02:57.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.674674 23035 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:02:57.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.675149 23035 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:02:57.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.709815 23035 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:02:57.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.710114 23035 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:02:57.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.710311 23035 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:02:57.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.710511 23035 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:02:57.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.710702 23035 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:02:57.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.710853 23035 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:44443/jwks.json
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.751770 23035 master_runner.cc:387] Master server non-default flags:
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/logs
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.85.20:45841
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/data
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/wal
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.85.62:38623,127.22.85.61:41445,127.22.85.60:41663
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.85.60:41663
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44443/jwks.json
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/data/info.pb
14:02:57.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.85.60
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cb747ff0ca0209e3fa98be4167c67edd474acd67
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Aug 2026 13:43:25 UTC on e7f111948823
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13227
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:02:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.753191 23035 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:02:57.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.754951 23035 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:02:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.759153 23033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 4
14:02:57.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.764351 23033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:57.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.767266 23045 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:02:57.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.784570 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } attempt: 7
14:02:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.817649 22956 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } has no permanent_uuid. Determining permanent_uuid...
14:02:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.835980 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:57.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.894603 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 1
14:02:57.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.903615 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:57.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:57.985247 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 2
14:02:57.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.993867 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:58.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:58.063892 23033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 5
14:02:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:58.072536 23033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:58.079452 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 3
14:02:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:58.087132 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:58.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:58.227722 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 4
14:02:58.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:58.236259 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:58.519904 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 5
14:02:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:58.528949 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:58.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:58.620110 23033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 6
14:02:58.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:58.645414 23033 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.081550 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 6
14:02:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:59.109184 22956 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.85.60:41663: Network error: Client connection negotiation failed: client connection to 127.22.85.60:41663: connect: Connection refused (error 111)
14:02:59.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:59.171315 23044 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee00 after lost signal to thread 23035
14:02:59.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:59.442000 23044 kernel_stack_watchdog.cc:198] Thread 23035 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
14:02:59.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:02:59.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:02:59.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:02:59.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:02:59.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:59.442497 23035 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.675s user 0.627s sys 1.035s
14:02:59.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:57.768293 23046 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:02:59.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:59.443002 23035 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.675s user 0.627s sys 1.035s
14:02:59.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:59.443585 23047 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1673 milliseconds
14:02:59.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:59.445053 23048 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:02:59.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.444932 23035 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:02:59.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.448589 23035 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:02:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.451290 23035 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:02:59.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.452702 23035 hybrid_clock.cc:648] HybridClock initialized: now 1786111379452664 us; error 37 us; skew 500 ppm
14:02:59.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.453639 23035 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:02:59.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.460047 23035 webserver.cc:533] Webserver started at http://127.22.85.60:32803/ using document root <none> and password file <none>
14:02:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.461071 23035 fs_manager.cc:362] Metadata directory not provided
14:02:59.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.461339 23035 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:02:59.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.461848 23035 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:02:59.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.466233 23035 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/data/instance:
14:02:59.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "31b7dcb87c5849f7bdd495149d4008b8"
14:02:59.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:02:59 on dist-test-slave-10pc"
14:02:59.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.467361 23035 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/wal/instance:
14:02:59.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "31b7dcb87c5849f7bdd495149d4008b8"
14:02:59.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:02:59 on dist-test-slave-10pc"
14:02:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.474936 23035 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.009s
14:02:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.480985 23061 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:02:59.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.482084 23035 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
14:02:59.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.482472 23035 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/data,/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/wal
14:02:59.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "31b7dcb87c5849f7bdd495149d4008b8"
14:02:59.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:02:59 on dist-test-slave-10pc"
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.482824 23035 fs_report.cc:389] FS layout report
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/wal
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/wal
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/data/data
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:02:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:02:59.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.540885 23035 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:02:59.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.542711 23035 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:02:59.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.543256 23035 kserver.cc:163] Server-wide thread pool size limit: 3276
14:02:59.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.620211 23035 rpc_server.cc:307] RPC server started. Bound to: 127.22.85.60:41663
14:02:59.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.620491 23115 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.85.60:41663 every 8 connection(s)
14:02:59.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.623250 23035 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/data/info.pb
14:02:59.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.629938 23116 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:02:59.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.631630 22868 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu as pid 23035
14:02:59.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.631983 22868 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/master-2/wal/instance
14:02:59.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.646081 23116 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } has no permanent_uuid. Determining permanent_uuid...
14:02:59.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.678962 23116 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } has no permanent_uuid. Determining permanent_uuid...
14:02:59.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.689971 23116 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } has no permanent_uuid. Determining permanent_uuid...
14:02:59.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.706080 23033 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 7
14:02:59.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.708963 23116 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8: Bootstrap starting.
14:02:59.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.716919 23116 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8: Neither blocks nor log segments found. Creating new log.
14:02:59.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.719681 23116 log.cc:826] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8: Log is configured to *not* fsync() on all Append() calls
14:02:59.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.726680 23116 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8: No bootstrap required, opened a new log
14:02:59.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.730618 23033 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d: Bootstrap starting.
14:02:59.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.738288 23033 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d: Neither blocks nor log segments found. Creating new log.
14:02:59.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.740542 23033 log.cc:826] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d: Log is configured to *not* fsync() on all Append() calls
14:02:59.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.746563 23033 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d: No bootstrap required, opened a new log
14:02:59.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.751387 23116 raft_consensus.cc:359] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } }
14:02:59.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.752760 23116 raft_consensus.cc:385] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:02:59.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.753093 23116 raft_consensus.cc:740] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31b7dcb87c5849f7bdd495149d4008b8, State: Initialized, Role: FOLLOWER
14:02:59.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.754110 23116 consensus_queue.cc:260] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } }
14:02:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.757373 23127 sys_catalog.cc:455] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } } }
14:02:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.758456 23127 sys_catalog.cc:458] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [sys.catalog]: This master's current role is: FOLLOWER
14:02:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.759403 23116 sys_catalog.cc:565] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [sys.catalog]: configured and running, proceeding with master startup.
14:02:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.775648 23033 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } }
14:02:59.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.776402 23033 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:02:59.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.776752 23033 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4a976288f24f47099dfaaeaa7210928d, State: Initialized, Role: FOLLOWER
14:02:59.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.777743 23033 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } }
14:02:59.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.780623 23137 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } } }
14:02:59.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.781483 23137 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [sys.catalog]: This master's current role is: FOLLOWER
14:02:59.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.782455 23033 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [sys.catalog]: configured and running, proceeding with master startup.
14:02:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:59.793277 23140 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:02:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:59.793646 23140 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:02:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:59.800844 23067 tablet.cc:2406] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:02:59.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.787555 23035 master_runner.cc:187] Error getting master registration for 127.22.85.62:38623: OK, instance_id { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" instance_seqno: 1786111375393164 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:02:59.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.821297 23035 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:02:59.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:59.836064 23152 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:02:59.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:02:59.836440 23152 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:02:59.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.831610 22958 master_runner.cc:187] Error getting master registration for 127.22.85.62:38623: OK, instance_id { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" instance_seqno: 1786111375393164 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:02:59.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:02:59.857195 22958 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:03:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:00.006485 22986 tablet.cc:2406] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.142777 22956 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } attempt: 7
14:03:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.142879 23137 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:00.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.143481 23137 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } }
14:03:00.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.151000 23090 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4a976288f24f47099dfaaeaa7210928d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31b7dcb87c5849f7bdd495149d4008b8" is_pre_election: true
14:03:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.152374 23090 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4a976288f24f47099dfaaeaa7210928d in term 0.
14:03:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.166218 22982 leader_election.cc:304] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 31b7dcb87c5849f7bdd495149d4008b8, 4a976288f24f47099dfaaeaa7210928d; no voters:
14:03:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.172340 23155 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [term 0 FOLLOWER]: Leader pre-election won for term 1
14:03:00.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.172829 23155 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:03:00.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.173172 23155 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [term 0 FOLLOWER]: Advancing to term 1
14:03:00.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.173226 23137 leader_election.cc:290] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 38b700491d2d4374aaf7ff824ee95a97 (127.22.85.62:38623), 31b7dcb87c5849f7bdd495149d4008b8 (127.22.85.60:41663)
14:03:00.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.181394 23155 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } }
14:03:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.186437 23090 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4a976288f24f47099dfaaeaa7210928d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31b7dcb87c5849f7bdd495149d4008b8"
14:03:00.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.187037 23090 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [term 0 FOLLOWER]: Advancing to term 1
14:03:00.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.188602 23155 leader_election.cc:290] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [CANDIDATE]: Term 1 election: Requested vote from peers 38b700491d2d4374aaf7ff824ee95a97 (127.22.85.62:38623), 31b7dcb87c5849f7bdd495149d4008b8 (127.22.85.60:41663)
14:03:00.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.201498 23090 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4a976288f24f47099dfaaeaa7210928d in term 1.
14:03:00.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.205119 22956 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97: Bootstrap starting.
14:03:00.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.208662 22982 leader_election.cc:304] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 31b7dcb87c5849f7bdd495149d4008b8, 4a976288f24f47099dfaaeaa7210928d; no voters:
14:03:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.209833 23155 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [term 1 FOLLOWER]: Leader election won for term 1
14:03:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.210582 23155 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [term 1 LEADER]: Becoming Leader. State: Replica: 4a976288f24f47099dfaaeaa7210928d, State: Running, Role: LEADER
14:03:00.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.213264 23155 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } }
14:03:00.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.223495 22956 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97: Neither blocks nor log segments found. Creating new log.
14:03:00.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.228559 22956 log.cc:826] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97: Log is configured to *not* fsync() on all Append() calls
14:03:00.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.235483 23137 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4a976288f24f47099dfaaeaa7210928d. Latest consensus state: current_term: 1 leader_uuid: "4a976288f24f47099dfaaeaa7210928d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } } }
14:03:00.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.236531 23137 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [sys.catalog]: This master's current role is: LEADER
14:03:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.279412 23158 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
14:03:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.278964 22956 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97: No bootstrap required, opened a new log
14:03:00.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.287576 23158 catalog_manager.cc:1520] Initializing Kudu cluster ID...
14:03:00.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.357913 23090 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [term 1 FOLLOWER]: Refusing update from remote peer 4a976288f24f47099dfaaeaa7210928d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:03:00.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.359843 23155 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [LEADER]: Connected to new peer: Peer: permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:03:00.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.400393 23160 sys_catalog.cc:455] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4a976288f24f47099dfaaeaa7210928d. Latest consensus state: current_term: 1 leader_uuid: "4a976288f24f47099dfaaeaa7210928d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } } }
14:03:00.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.401036 23159 mvcc.cc:204] Tried to move back new op lower bound from 7315912213903380480 to 7315912213369597952. Current Snapshot: MvccSnapshot[applied={T|T < 7315912213903380480}]
14:03:00.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.401264 23160 sys_catalog.cc:458] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [sys.catalog]: This master's current role is: FOLLOWER
14:03:00.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.407831 23161 mvcc.cc:204] Tried to move back new op lower bound from 7315912213903380480 to 7315912213369597952. Current Snapshot: MvccSnapshot[applied={T|T < 7315912213903380480}]
14:03:00.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.409370 22956 raft_consensus.cc:359] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } }
14:03:00.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.410286 22956 raft_consensus.cc:385] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:00.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.412242 22956 raft_consensus.cc:740] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 38b700491d2d4374aaf7ff824ee95a97, State: Initialized, Role: FOLLOWER
14:03:00.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.414307 22956 consensus_queue.cc:260] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } }
14:03:00.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.416010 23137 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4a976288f24f47099dfaaeaa7210928d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } } }
14:03:00.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.416836 23137 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [sys.catalog]: This master's current role is: LEADER
14:03:00.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.419623 23160 sys_catalog.cc:455] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4a976288f24f47099dfaaeaa7210928d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } } }
14:03:00.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.420495 23160 sys_catalog.cc:458] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8 [sys.catalog]: This master's current role is: FOLLOWER
14:03:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.423489 23158 catalog_manager.cc:1383] Generated new cluster ID: abfbf5aa05594fdaafd7acbe69c91927
14:03:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.423796 23158 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
14:03:00.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.447695 22956 sys_catalog.cc:565] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97 [sys.catalog]: configured and running, proceeding with master startup.
14:03:00.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.447885 22928 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4a976288f24f47099dfaaeaa7210928d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "38b700491d2d4374aaf7ff824ee95a97" is_pre_election: true
14:03:00.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.445750 23165 sys_catalog.cc:455] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } } }
14:03:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.450042 22926 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97 [term 0 FOLLOWER]: Advancing to term 1
14:03:00.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.450289 22927 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4a976288f24f47099dfaaeaa7210928d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "38b700491d2d4374aaf7ff824ee95a97"
14:03:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.450594 23165 sys_catalog.cc:458] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97 [sys.catalog]: This master's current role is: FOLLOWER
14:03:00.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.460057 22926 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97 [term 1 FOLLOWER]: Refusing update from remote peer 4a976288f24f47099dfaaeaa7210928d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:03:00.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.462085 23155 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [LEADER]: Connected to new peer: Peer: permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:03:00.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.512120 23158 catalog_manager.cc:1406] Generated new certificate authority record
14:03:00.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.522837 23158 catalog_manager.cc:1540] Loading token signing keys...
14:03:00.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.525964 23165 sys_catalog.cc:455] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4a976288f24f47099dfaaeaa7210928d. Latest consensus state: current_term: 1 leader_uuid: "4a976288f24f47099dfaaeaa7210928d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } } }
14:03:00.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.528069 23173 sys_catalog.cc:455] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4a976288f24f47099dfaaeaa7210928d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } } }
14:03:00.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.529209 23173 sys_catalog.cc:458] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97 [sys.catalog]: This master's current role is: FOLLOWER
14:03:00.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.527657 23165 sys_catalog.cc:458] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97 [sys.catalog]: This master's current role is: FOLLOWER
14:03:00.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.533691 23137 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4a976288f24f47099dfaaeaa7210928d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "38b700491d2d4374aaf7ff824ee95a97" member_type: VOTER last_known_addr { host: "127.22.85.62" port: 38623 } } peers { permanent_uuid: "4a976288f24f47099dfaaeaa7210928d" member_type: VOTER last_known_addr { host: "127.22.85.61" port: 41445 } } peers { permanent_uuid: "31b7dcb87c5849f7bdd495149d4008b8" member_type: VOTER last_known_addr { host: "127.22.85.60" port: 41663 } } }
14:03:00.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.534390 23137 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d [sys.catalog]: This master's current role is: LEADER
14:03:00.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.574951 23158 catalog_manager.cc:6070] T 00000000000000000000000000000000 P 4a976288f24f47099dfaaeaa7210928d: Generated new TSK 0
14:03:00.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.576124 23158 catalog_manager.cc:1550] Initializing in-progress tserver states...
14:03:00.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.590752 23180 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:03:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.618790 22868 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/wal
14:03:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/data
14:03:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/logs
14:03:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/data/info.pb
14:03:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.85.1:0
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.85.1
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.85.1
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.85.62:38623,127.22.85.61:41445,127.22.85.60:41663
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.85.20:45841
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:03:00.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.799817 23140 catalog_manager.cc:1295] Loaded cluster ID: abfbf5aa05594fdaafd7acbe69c91927
14:03:00.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.800351 23140 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8: loading cluster ID for follower catalog manager: success
14:03:00.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.808271 23140 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8: acquiring CA information for follower catalog manager: success
14:03:00.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:00.814476 23140 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 31b7dcb87c5849f7bdd495149d4008b8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:03:01.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:01.098397 23181 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:03:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:01.099090 23181 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:01.099982 23181 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:03:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:01.159533 23181 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:03:01.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:01.160957 23181 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.85.1
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:01.227685 23181 tablet_server_runner.cc:78] Tablet server non-default flags:
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/logs
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.85.20:45841
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/data
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/wal
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.85.1:0
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/data/info.pb
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.85.1
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.85.62:38623,127.22.85.61:41445,127.22.85.60:41663
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.85.1
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cb747ff0ca0209e3fa98be4167c67edd474acd67
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Aug 2026 13:43:25 UTC on e7f111948823
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13227
14:03:01.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:03:01.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:01.230432 23181 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:01.232882 23181 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:03:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:01.249209 23187 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:01.613247 23180 catalog_manager.cc:1295] Loaded cluster ID: abfbf5aa05594fdaafd7acbe69c91927
14:03:01.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:01.614871 23180 catalog_manager.cc:1588] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97: loading cluster ID for follower catalog manager: success
14:03:01.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:01.637437 23180 catalog_manager.cc:1610] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97: acquiring CA information for follower catalog manager: success
14:03:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:01.647702 23180 catalog_manager.cc:1638] T 00000000000000000000000000000000 P 38b700491d2d4374aaf7ff824ee95a97: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:03:03.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:01.252502 23188 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:03.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:03.299031 23181 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.050s user 0.713s sys 1.057s
14:03:03.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:03.300659 23181 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.052s user 0.714s sys 1.057s
14:03:03.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:02.651165 23186 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 23181
14:03:03.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.313313 23181 server_base.cc:1061] running on GCE node
14:03:03.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:03.314538 23194 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:03.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.316222 23181 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:03.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.324726 23181 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:03.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.326646 23181 hybrid_clock.cc:648] HybridClock initialized: now 1786111383326564 us; error 88 us; skew 500 ppm
14:03:03.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.327991 23181 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.341574 23181 webserver.cc:533] Webserver started at http://127.22.85.1:39195/ using document root <none> and password file <none>
14:03:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.343078 23181 fs_manager.cc:362] Metadata directory not provided
14:03:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.343505 23181 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.344188 23181 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:03:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.350857 23181 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/data/instance:
14:03:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cf4b74a57e8642548718a1bd0f0520c3"
14:03:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:03 on dist-test-slave-10pc"
14:03:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.353279 23181 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/wal/instance:
14:03:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cf4b74a57e8642548718a1bd0f0520c3"
14:03:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:03 on dist-test-slave-10pc"
14:03:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.365481 23181 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.004s sys 0.004s
14:03:03.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.374099 23199 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.375609 23181 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
14:03:03.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.376217 23181 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/data,/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/wal
14:03:03.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cf4b74a57e8642548718a1bd0f0520c3"
14:03:03.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:03 on dist-test-slave-10pc"
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.376837 23181 fs_report.cc:389] FS layout report
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/wal
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/wal
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/data/data
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:03.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.477546 23181 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:03.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.482513 23181 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:03.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.483331 23181 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.487694 23181 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:03:03.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.494990 23181 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:03:03.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.495275 23181 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:03:03.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.495581 23181 ts_tablet_manager.cc:616] Registered 0 tablets
14:03:03.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.495792 23181 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:03:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.679796 23181 rpc_server.cc:307] RPC server started. Bound to: 127.22.85.1:38207
14:03:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.679863 23315 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.85.1:38207 every 8 connection(s)
14:03:03.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.684796 23181 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/data/info.pb
14:03:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.691308 22868 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu as pid 23181
14:03:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.692050 22868 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-0/wal/instance
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.709697 22868 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/wal
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/data
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/logs
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/data/info.pb
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:03.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:03:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.85.2:0
14:03:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.85.2
14:03:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.85.2
14:03:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.85.62:38623,127.22.85.61:41445,127.22.85.60:41663
14:03:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.85.20:45841
14:03:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:03:03.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.756007 23316 heartbeater.cc:344] Connected to a master server at 127.22.85.60:41663
14:03:03.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.756596 23316 heartbeater.cc:461] Registering TS with master...
14:03:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.756894 23318 heartbeater.cc:344] Connected to a master server at 127.22.85.62:38623
14:03:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.757296 23318 heartbeater.cc:461] Registering TS with master...
14:03:03.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.758123 23316 heartbeater.cc:507] Master 127.22.85.60:41663 requested a full tablet report, sending...
14:03:03.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.758694 23318 heartbeater.cc:507] Master 127.22.85.62:38623 requested a full tablet report, sending...
14:03:03.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.762593 23079 ts_manager.cc:194] Registered new tserver with Master: cf4b74a57e8642548718a1bd0f0520c3 (127.22.85.1:38207)
14:03:03.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.763648 22917 ts_manager.cc:194] Registered new tserver with Master: cf4b74a57e8642548718a1bd0f0520c3 (127.22.85.1:38207)
14:03:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.768872 23317 heartbeater.cc:344] Connected to a master server at 127.22.85.61:41445
14:03:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.769264 23317 heartbeater.cc:461] Registering TS with master...
14:03:03.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.770185 23317 heartbeater.cc:507] Master 127.22.85.61:41445 requested a full tablet report, sending...
14:03:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.773070 22998 ts_manager.cc:194] Registered new tserver with Master: cf4b74a57e8642548718a1bd0f0520c3 (127.22.85.1:38207)
14:03:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:03.775833 22998 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.85.1:38777
14:03:04.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:04.079715 23326 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:03:04.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:04.080212 23326 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:04.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:04.080690 23326 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:03:04.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:04.117053 23326 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:03:04.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:04.117848 23326 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.85.2
14:03:04.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:04.155485 23326 tablet_server_runner.cc:78] Tablet server non-default flags:
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/logs
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.85.20:45841
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/data
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/wal
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.85.2:0
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/data/info.pb
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.85.2
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.85.62:38623,127.22.85.61:41445,127.22.85.60:41663
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.85.2
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cb747ff0ca0209e3fa98be4167c67edd474acd67
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Aug 2026 13:43:25 UTC on e7f111948823
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13227
14:03:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:03:04.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:04.156991 23326 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:04.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:04.158619 23326 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:03:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:04.172504 23332 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:04.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:04.780339 23317 heartbeater.cc:499] Master 127.22.85.61:41445 was elected leader, sending a full tablet report...
14:03:05.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:04.177067 23335 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:05.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:04.177255 23333 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:05.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:04.177729 23326 server_base.cc:1061] running on GCE node
14:03:05.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.448151 23326 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.450611 23326 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:05.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.452032 23326 hybrid_clock.cc:648] HybridClock initialized: now 1786111385451985 us; error 49 us; skew 500 ppm
14:03:05.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.453125 23326 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:05.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.460341 23326 webserver.cc:533] Webserver started at http://127.22.85.2:39479/ using document root <none> and password file <none>
14:03:05.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.461508 23326 fs_manager.cc:362] Metadata directory not provided
14:03:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.461786 23326 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:05.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.462342 23326 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:03:05.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.467006 23326 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/data/instance:
14:03:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "983bc0c1860d4fe992eb0579e8530a56"
14:03:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:05 on dist-test-slave-10pc"
14:03:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.468379 23326 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/wal/instance:
14:03:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "983bc0c1860d4fe992eb0579e8530a56"
14:03:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:05 on dist-test-slave-10pc"
14:03:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.476166 23326 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
14:03:05.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.482378 23345 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:05.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.483526 23326 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:03:05.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.483927 23326 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/data,/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/wal
14:03:05.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "983bc0c1860d4fe992eb0579e8530a56"
14:03:05.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:05 on dist-test-slave-10pc"
14:03:05.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.484364 23326 fs_report.cc:389] FS layout report
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/wal
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/wal
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/data/data
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:05.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.539029 23326 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.540755 23326 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.541189 23326 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:05.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.543900 23326 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:03:05.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.548417 23326 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:03:05.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.548638 23326 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
14:03:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.548874 23326 ts_tablet_manager.cc:616] Registered 0 tablets
14:03:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.549012 23326 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:03:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.702608 23326 rpc_server.cc:307] RPC server started. Bound to: 127.22.85.2:35715
14:03:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.702736 23459 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.85.2:35715 every 8 connection(s)
14:03:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.705219 23326 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/data/info.pb
14:03:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.710942 22868 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu as pid 23326
14:03:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.711488 22868 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-1/wal/instance
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.724440 22868 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/wal
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/data
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/logs
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/data/info.pb
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.85.3:0
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.85.3
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.85.3
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.85.62:38623,127.22.85.61:41445,127.22.85.60:41663
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.85.20:45841
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:03:05.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.751879 23461 heartbeater.cc:344] Connected to a master server at 127.22.85.61:41445
14:03:05.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.752620 23461 heartbeater.cc:461] Registering TS with master...
14:03:05.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.752790 23460 heartbeater.cc:344] Connected to a master server at 127.22.85.60:41663
14:03:05.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.753198 23460 heartbeater.cc:461] Registering TS with master...
14:03:05.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.754570 23462 heartbeater.cc:344] Connected to a master server at 127.22.85.62:38623
14:03:05.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.754611 23461 heartbeater.cc:507] Master 127.22.85.61:41445 requested a full tablet report, sending...
14:03:05.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.754613 23460 heartbeater.cc:507] Master 127.22.85.60:41663 requested a full tablet report, sending...
14:03:05.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.755016 23462 heartbeater.cc:461] Registering TS with master...
14:03:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.756021 23462 heartbeater.cc:507] Master 127.22.85.62:38623 requested a full tablet report, sending...
14:03:05.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.757880 22998 ts_manager.cc:194] Registered new tserver with Master: 983bc0c1860d4fe992eb0579e8530a56 (127.22.85.2:35715)
14:03:05.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.758307 23079 ts_manager.cc:194] Registered new tserver with Master: 983bc0c1860d4fe992eb0579e8530a56 (127.22.85.2:35715)
14:03:05.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.759658 22998 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.85.2:55015
14:03:05.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:05.760948 22917 ts_manager.cc:194] Registered new tserver with Master: 983bc0c1860d4fe992eb0579e8530a56 (127.22.85.2:35715)
14:03:06.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:06.082221 23470 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:03:06.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:06.082734 23470 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:06.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:06.083184 23470 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:03:06.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:06.119263 23470 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:03:06.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:06.120222 23470 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.85.3
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:06.158147 23470 tablet_server_runner.cc:78] Tablet server non-default flags:
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/logs
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.85.20:45841
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/data
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/wal
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.85.3:0
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/data/info.pb
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.85.3
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.85.62:38623,127.22.85.61:41445,127.22.85.60:41663
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:06.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.85.3
14:03:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:03:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:03:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cb747ff0ca0209e3fa98be4167c67edd474acd67
14:03:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Aug 2026 13:43:25 UTC on e7f111948823
14:03:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13227
14:03:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:03:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:06.159497 23470 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:06.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:06.161298 23470 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:03:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:06.175024 23477 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:06.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:06.764349 23461 heartbeater.cc:499] Master 127.22.85.61:41445 was elected leader, sending a full tablet report...
14:03:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:07.578703 23476 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 23470
14:03:07.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:07.767395 23470 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.592s user 0.463s sys 1.080s
14:03:07.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:07.767901 23470 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.592s user 0.463s sys 1.080s
14:03:07.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:06.176630 23479 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:07.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:07.769330 23480 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1592 milliseconds
14:03:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:07.770586 23481 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.770505 23470 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:03:07.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.777150 23470 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:07.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.779443 23470 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:07.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.780874 23470 hybrid_clock.cc:648] HybridClock initialized: now 1786111387780823 us; error 39 us; skew 500 ppm
14:03:07.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.782097 23470 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:07.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.792801 23470 webserver.cc:533] Webserver started at http://127.22.85.3:37793/ using document root <none> and password file <none>
14:03:07.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.794207 23470 fs_manager.cc:362] Metadata directory not provided
14:03:07.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.794536 23470 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:07.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.795136 23470 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:03:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.800231 23470 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/data/instance:
14:03:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6260f17624f54ee6a5005150237f4375"
14:03:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:07 on dist-test-slave-10pc"
14:03:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.801519 23470 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/wal/instance:
14:03:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6260f17624f54ee6a5005150237f4375"
14:03:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:07 on dist-test-slave-10pc"
14:03:07.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.809499 23470 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.002s
14:03:07.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.815879 23489 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.817111 23470 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
14:03:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.817484 23470 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/data,/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/wal
14:03:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6260f17624f54ee6a5005150237f4375"
14:03:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:07 on dist-test-slave-10pc"
14:03:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.817869 23470 fs_report.cc:389] FS layout report
14:03:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/wal
14:03:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/wal
14:03:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/data/data
14:03:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.871572 23470 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.873297 23470 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.873816 23470 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:07.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.876677 23470 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:03:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.881088 23470 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:03:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.881325 23470 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:03:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.881654 23470 ts_tablet_manager.cc:616] Registered 0 tablets
14:03:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:07.881843 23470 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:03:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.027957 23470 rpc_server.cc:307] RPC server started. Bound to: 127.22.85.3:41789
14:03:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.028087 23606 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.85.3:41789 every 8 connection(s)
14:03:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.030696 23470 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/data/info.pb
14:03:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.038501 22868 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu as pid 23470
14:03:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.039136 22868 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5523923200326095965/ts-2/wal/instance
14:03:08.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.086140 23608 heartbeater.cc:344] Connected to a master server at 127.22.85.61:41445
14:03:08.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.086702 23608 heartbeater.cc:461] Registering TS with master...
14:03:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.088131 23608 heartbeater.cc:507] Master 127.22.85.61:41445 requested a full tablet report, sending...
14:03:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.088279 23607 heartbeater.cc:344] Connected to a master server at 127.22.85.60:41663
14:03:08.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.088686 23607 heartbeater.cc:461] Registering TS with master...
14:03:08.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.089622 23607 heartbeater.cc:507] Master 127.22.85.60:41663 requested a full tablet report, sending...
14:03:08.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.091317 22998 ts_manager.cc:194] Registered new tserver with Master: 6260f17624f54ee6a5005150237f4375 (127.22.85.3:41789)
14:03:08.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.092686 23609 heartbeater.cc:344] Connected to a master server at 127.22.85.62:38623
14:03:08.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.093014 23609 heartbeater.cc:461] Registering TS with master...
14:03:08.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.093295 23079 ts_manager.cc:194] Registered new tserver with Master: 6260f17624f54ee6a5005150237f4375 (127.22.85.3:41789)
14:03:08.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.093355 22998 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.22.85.3:39027
14:03:08.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.093927 23609 heartbeater.cc:507] Master 127.22.85.62:38623 requested a full tablet report, sending...
14:03:08.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.096930 22916 ts_manager.cc:194] Registered new tserver with Master: 6260f17624f54ee6a5005150237f4375 (127.22.85.3:41789)
14:03:08.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.103384 22868 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
14:03:08.115 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:03:08.115 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:03:08.117 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.85.62"
port: 38623
}
bound_http_address {
host: "127.22.85.62"
port: 41951
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.85.61"
port: 41445
}
bound_http_address {
host: "127.22.85.61"
port: 39263
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.85.60"
port: 41663
}
bound_http_address {
host: "127.22.85.60"
port: 32803
}
}
}
14:03:08.117 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:03:08.118 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.22.85.1"
port: 38207
}
bound_http_address {
host: "127.22.85.1"
port: 39195
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.22.85.2"
port: 35715
}
bound_http_address {
host: "127.22.85.2"
port: 39479
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.22.85.3"
port: 41789
}
bound_http_address {
host: "127.22.85.3"
port: 37793
}
}
}
14:03:08.118 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:03:08.119 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster6785716446432931129/krb5kdc/krb5cc
14:03:08.119 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:03:08.120 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task_tJAxU/test-tmp
14:03:08.121 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:03:08.121 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task_tJAxU/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:03:08.124 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:03:08.470 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:08.471 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:03:08.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.474462 23619 mini_oidc.cc:150] Starting JWKS server
14:03:08.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.477824 23619 webserver.cc:533] Webserver started at http://127.0.0.1:45369/ using document root <none> and password file <none>
14:03:08.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.478358 23619 mini_oidc.cc:186] Starting OIDC Discovery server
14:03:08.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:08.480159 23619 webserver.cc:533] Webserver started at http://127.0.0.1:34557/ using document root <none> and password file <none>
14:03:08.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:03:08.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/principal' for realm 'KRBTEST.COM',
14:03:08.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:03:08.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:08 dist-test-slave-10pc krb5kdc[23630](info): setting up network...
14:03:08.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:03:08.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:08 dist-test-slave-10pc krb5kdc[23630](info): set up 2 sockets
14:03:08.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:08 dist-test-slave-10pc krb5kdc[23630](info): commencing operation
14:03:08.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:03:09.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:09.097297 23608 heartbeater.cc:499] Master 127.22.85.61:41445 was elected leader, sending a full tablet report...
14:03:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:10.671788 23619 mini_kdc.cc:121] Time spent starting KDC: real 2.153s user 0.005s sys 0.016s
14:03:10.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:03:10.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:03:10.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:03:10.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:03:10.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:03:10.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:03:10.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:03:10.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:03:10.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:03:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:03:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/test-user.keytab.
14:03:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/test-user.keytab.
14:03:10.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:10 dist-test-slave-10pc krb5kdc[23630](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111390, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:03:10.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:03:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T14:03:10Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:03:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T14:03:10Z Disabled control of system clock
14:03:10.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.16.254@KRBTEST.COM; defaulting to no policy
14:03:10.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:03:10.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.16.254@KRBTEST.COM" created.
14:03:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:03:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.16.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab.
14:03:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.16.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab.
14:03:10.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.16.254@KRBTEST.COM; defaulting to no policy
14:03:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:03:10.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.16.254@KRBTEST.COM" created.
14:03:10.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:03:10.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.16.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab.
14:03:10.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.16.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab.
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:10.921928 23619 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/logs
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data/info.pb
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:10.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.16.254:35195
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.16.254
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.16.254
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.16.212:35199
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.16.254:35195,127.23.16.253:46275,127.23.16.252:38825
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45369/jwks.json
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:03:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:03:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:11.252805 23646 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:03:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:11.253456 23646 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:11.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:11.254127 23646 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:03:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:11.287861 23646 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:03:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:11.288111 23646 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:03:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:11.288376 23646 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:03:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:11.288594 23646 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:03:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:11.288791 23646 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:03:11.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:11.288982 23646 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:03:11.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:11.289136 23646 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45369/jwks.json
14:03:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:11.334844 23646 master_runner.cc:387] Master server non-default flags:
14:03:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/logs
14:03:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.16.212:35199
14:03:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data
14:03:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal
14:03:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.16.254:35195,127.23.16.253:46275,127.23.16.252:38825
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.16.254
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.16.254:35195
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45369/jwks.json
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data/info.pb
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.16.254
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cb747ff0ca0209e3fa98be4167c67edd474acd67
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Aug 2026 13:43:25 UTC on e7f111948823
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13227
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:03:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:11.336160 23646 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:11.337244 23646 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:03:11.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:11.349574 23653 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:12.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:11.351240 23654 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:12.653736 23655 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
14:03:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:12.655059 23656 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:12.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:12.658470 23646 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.308s user 0.474s sys 0.831s
14:03:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:12.658780 23646 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.308s user 0.474s sys 0.831s
14:03:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.659049 23646 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:03:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.660486 23646 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:12.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.663767 23646 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:12.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.665189 23646 hybrid_clock.cc:648] HybridClock initialized: now 1786111392665140 us; error 65 us; skew 500 ppm
14:03:12.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.666312 23646 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:12.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:12 dist-test-slave-10pc krb5kdc[23630](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111392, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:03:12.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.675638 23646 init.cc:377] Logged in from keytab as kudu/127.23.16.254@KRBTEST.COM (short username kudu)
14:03:12.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.682137 23646 webserver.cc:533] Webserver started at http://127.23.16.254:33053/ using document root <none> and password file <none>
14:03:12.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.683410 23646 fs_manager.cc:362] Metadata directory not provided
14:03:12.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.683699 23646 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:12.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.684267 23646 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:03:12.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.690500 23646 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data/instance:
14:03:12.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "444404fe730945b1a70ccfef9b90f390"
14:03:12.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:12 on dist-test-slave-10pc"
14:03:12.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.692293 23646 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal/instance:
14:03:12.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "444404fe730945b1a70ccfef9b90f390"
14:03:12.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:12 on dist-test-slave-10pc"
14:03:12.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.701433 23646 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.004s sys 0.003s
14:03:12.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.709271 23664 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:12.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.710846 23646 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
14:03:12.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.711241 23646 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data,/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal
14:03:12.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "444404fe730945b1a70ccfef9b90f390"
14:03:12.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:12 on dist-test-slave-10pc"
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.711681 23646 fs_report.cc:389] FS layout report
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data/data
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:12.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:12.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.789047 23646 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.800890 23646 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.801306 23646 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.881762 23646 rpc_server.cc:307] RPC server started. Bound to: 127.23.16.254:35195
14:03:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.881873 23716 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.16.254:35195 every 8 connection(s)
14:03:12.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.884742 23646 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data/info.pb
14:03:12.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.891248 23717 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:12.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.894193 23619 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu as pid 23646
14:03:12.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.894829 23619 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal/instance
14:03:12.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.909303 23717 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } has no permanent_uuid. Determining permanent_uuid...
14:03:12.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.16.253@KRBTEST.COM; defaulting to no policy
14:03:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:03:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.16.253@KRBTEST.COM" created.
14:03:12.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:12 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111392, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.254@KRBTEST.COM for kudu/127.23.16.254@KRBTEST.COM
14:03:12.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:03:12.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.16.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab.
14:03:12.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.16.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab.
14:03:12.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.960137 23717 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } has no permanent_uuid. Determining permanent_uuid...
14:03:12.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:12.962749 23668 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:12.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.16.253@KRBTEST.COM; defaulting to no policy
14:03:12.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:12.967734 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.253:46275: Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:03:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.16.253@KRBTEST.COM" created.
14:03:12.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:03:12.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.16.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab.
14:03:12.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.16.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab.
14:03:12.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:12.996377 23619 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:12.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:12.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal
14:03:12.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/logs
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data/info.pb
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.16.253:46275
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.16.253
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.16.253
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.16.212:35199
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.16.254:35195,127.23.16.253:46275,127.23.16.252:38825
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:12.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45369/jwks.json
14:03:12.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:03:12.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:03:13.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:13.017318 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } attempt: 1
14:03:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.022243 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.253:46275: Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:13.090684 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } attempt: 2
14:03:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.095561 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.253:46275: Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:13.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:13.186945 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } attempt: 3
14:03:13.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.191848 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.253:46275: Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.325898 23726 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:03:13.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.326618 23726 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:13.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.327430 23726 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:03:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:13.335297 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } attempt: 4
14:03:13.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.340390 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.253:46275: Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:13.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.365402 23726 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:03:13.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.365715 23726 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:03:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.366149 23726 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:03:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.366427 23726 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:03:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.366647 23726 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:03:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.366842 23726 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:03:13.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.366988 23726 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45369/jwks.json
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:13.408900 23726 master_runner.cc:387] Master server non-default flags:
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/logs
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.16.212:35199
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.16.254:35195,127.23.16.253:46275,127.23.16.252:38825
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.16.253
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.16.253:46275
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45369/jwks.json
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data/info.pb
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.16.253
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cb747ff0ca0209e3fa98be4167c67edd474acd67
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Aug 2026 13:43:25 UTC on e7f111948823
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13227
14:03:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:03:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:13.410341 23726 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:13.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:13.411521 23726 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:03:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.425911 23733 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:13.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:13.639879 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } attempt: 5
14:03:13.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.651405 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.253:46275: Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.199052 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } attempt: 6
14:03:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:14.211541 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.253:46275: Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.430756 23734 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:13.431142 23736 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:14.798645 23735 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1368 milliseconds
14:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.798797 23726 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:03:14.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.800172 23726 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:14.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.803519 23726 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.804966 23726 hybrid_clock.cc:648] HybridClock initialized: now 1786111394804921 us; error 38 us; skew 500 ppm
14:03:14.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.806006 23726 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:14.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:14 dist-test-slave-10pc krb5kdc[23630](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111394, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:03:14.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.812996 23726 init.cc:377] Logged in from keytab as kudu/127.23.16.253@KRBTEST.COM (short username kudu)
14:03:14.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.819185 23726 webserver.cc:533] Webserver started at http://127.23.16.253:38921/ using document root <none> and password file <none>
14:03:14.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.820713 23726 fs_manager.cc:362] Metadata directory not provided
14:03:14.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.821002 23726 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:14.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.821580 23726 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:03:14.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.825990 23726 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data/instance:
14:03:14.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "feb1147a9db74cca933cbaf1af70b146"
14:03:14.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:14 on dist-test-slave-10pc"
14:03:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.827167 23726 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal/instance:
14:03:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "feb1147a9db74cca933cbaf1af70b146"
14:03:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:14 on dist-test-slave-10pc"
14:03:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.836879 23726 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.005s
14:03:14.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.842679 23749 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:14.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.843859 23726 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
14:03:14.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.844242 23726 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data,/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal
14:03:14.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "feb1147a9db74cca933cbaf1af70b146"
14:03:14.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:14 on dist-test-slave-10pc"
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.844668 23726 fs_report.cc:389] FS layout report
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data/data
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:14.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.907617 23726 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:14.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.913986 23726 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:14.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.914548 23726 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:14.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.993526 23726 rpc_server.cc:307] RPC server started. Bound to: 127.23.16.253:46275
14:03:14.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.993631 23802 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.16.253:46275 every 8 connection(s)
14:03:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:14.996546 23726 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data/info.pb
14:03:15.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.001250 23619 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu as pid 23726
14:03:15.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.001775 23619 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal/instance
14:03:15.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.003630 23803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:15.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.020282 23803 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } has no permanent_uuid. Determining permanent_uuid...
14:03:15.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.23.16.252@KRBTEST.COM; defaulting to no policy
14:03:15.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:03:15.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.23.16.252@KRBTEST.COM" created.
14:03:15.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:15 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111394, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.253@KRBTEST.COM for kudu/127.23.16.254@KRBTEST.COM
14:03:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:03:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.16.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab.
14:03:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.23.16.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab.
14:03:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.064714 23803 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } has no permanent_uuid. Determining permanent_uuid...
14:03:15.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:15 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111394, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.253@KRBTEST.COM for kudu/127.23.16.253@KRBTEST.COM
14:03:15.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.23.16.252@KRBTEST.COM; defaulting to no policy
14:03:15.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:03:15.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.23.16.252@KRBTEST.COM" created.
14:03:15.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.093605 23803 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } has no permanent_uuid. Determining permanent_uuid...
14:03:15.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.096107 23750 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:15.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.100328 23803 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:03:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.16.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab.
14:03:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.23.16.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab.
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.116909 23619 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/wal
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/logs
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data/info.pb
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.16.252:38825
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.16.252
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.16.252
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.16.212:35199
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.16.254:35195,127.23.16.253:46275,127.23.16.252:38825
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45369/jwks.json
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:03:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:03:15.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.149848 23803 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 1
14:03:15.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.154645 23803 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.223078 23803 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 2
14:03:15.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.227943 23803 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:15.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.272282 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } attempt: 7
14:03:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:15 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111392, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.254@KRBTEST.COM for kudu/127.23.16.253@KRBTEST.COM
14:03:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.298496 23717 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } has no permanent_uuid. Determining permanent_uuid...
14:03:15.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.304762 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.319365 23803 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 3
14:03:15.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.324568 23803 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:15.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.363299 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 1
14:03:15.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.367902 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:15.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.449285 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 2
14:03:15.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.453733 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:15.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.453858 23813 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:03:15.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.454658 23813 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:15.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.455396 23813 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:03:15.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.467988 23803 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 4
14:03:15.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.472584 23803 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:15.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.491470 23813 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:03:15.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.491714 23813 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:03:15.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.491978 23813 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:03:15.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.492197 23813 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:03:15.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.492393 23813 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:03:15.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.492611 23813 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:03:15.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.492761 23813 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45369/jwks.json
14:03:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.533365 23813 master_runner.cc:387] Master server non-default flags:
14:03:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/logs
14:03:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.16.212:35199
14:03:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/wal
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.16.254:35195,127.23.16.253:46275,127.23.16.252:38825
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.16.252
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.16.252:38825
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45369/jwks.json
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data/info.pb
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.16.252
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cb747ff0ca0209e3fa98be4167c67edd474acd67
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Aug 2026 13:43:25 UTC on e7f111948823
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13227
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:03:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.534746 23813 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.535759 23813 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:03:15.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.539106 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 3
14:03:15.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.543978 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:15.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.548094 23821 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:15.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.684486 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 4
14:03:15.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.692687 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:15.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.772045 23803 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 5
14:03:15.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.783710 23803 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:15.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:15.976302 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 5
14:03:15.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.985536 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:16.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:16.331259 23803 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 6
14:03:16.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:16.370384 23803 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:16.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:16.538148 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 6
14:03:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:16.577925 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:17.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.432680 23803 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 7
14:03:17.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:16.955931 23820 debug-util.cc:398] Leaking SignalData structure 0x7b0800028cc0 after lost signal to thread 23813
14:03:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:17.466535 23813 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.917s user 0.001s sys 0.000s
14:03:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:17.467783 23813 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.919s user 0.002s sys 0.001s
14:03:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:15.549721 23822 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:17.467243 23823 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1916 milliseconds
14:03:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:17.469673 23824 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.469743 23813 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:03:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:17.472499 23803 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.475965 23813 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:17.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.478606 23813 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:17.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.480185 23813 hybrid_clock.cc:648] HybridClock initialized: now 1786111397480128 us; error 48 us; skew 500 ppm
14:03:17.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.481129 23813 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:17.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:17 dist-test-slave-10pc krb5kdc[23630](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111397, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:03:17.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.487986 23813 init.cc:377] Logged in from keytab as kudu/127.23.16.252@KRBTEST.COM (short username kudu)
14:03:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.493496 23813 webserver.cc:533] Webserver started at http://127.23.16.252:44041/ using document root <none> and password file <none>
14:03:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.494391 23813 fs_manager.cc:362] Metadata directory not provided
14:03:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.494644 23813 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:17.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.495188 23813 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:03:17.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.499584 23813 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data/instance:
14:03:17.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db3cc88e701c4931bfed4ebff3b7965f"
14:03:17.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:17 on dist-test-slave-10pc"
14:03:17.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.500770 23813 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/wal/instance:
14:03:17.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db3cc88e701c4931bfed4ebff3b7965f"
14:03:17.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:17 on dist-test-slave-10pc"
14:03:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.508829 23813 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
14:03:17.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.514983 23842 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.516158 23813 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
14:03:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.516610 23813 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data,/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/wal
14:03:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db3cc88e701c4931bfed4ebff3b7965f"
14:03:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:17 on dist-test-slave-10pc"
14:03:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.517017 23813 fs_report.cc:389] FS layout report
14:03:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/wal
14:03:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/wal
14:03:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data/data
14:03:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:17.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.606814 23813 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:17.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.611472 23717 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } attempt: 7
14:03:17.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.614498 23813 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:17.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.614986 23813 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:17.618571 23717 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.23.16.252:38825: Network error: Client connection negotiation failed: client connection to 127.23.16.252:38825: connect: Connection refused (error 111)
14:03:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.698957 23813 rpc_server.cc:307] RPC server started. Bound to: 127.23.16.252:38825
14:03:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.699043 23897 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.16.252:38825 every 8 connection(s)
14:03:17.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.701843 23813 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data/info.pb
14:03:17.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.707808 23619 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu as pid 23813
14:03:17.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.708303 23619 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/wal/instance
14:03:17.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.709321 23898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:03:17.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.724196 23898 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } has no permanent_uuid. Determining permanent_uuid...
14:03:17.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:17 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111390, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.16.252@KRBTEST.COM
14:03:17.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:17 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111397, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.252@KRBTEST.COM for kudu/127.23.16.254@KRBTEST.COM
14:03:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.759094 23898 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } has no permanent_uuid. Determining permanent_uuid...
14:03:17.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:17 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111390, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.16.253@KRBTEST.COM
14:03:17.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:17 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111397, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.252@KRBTEST.COM for kudu/127.23.16.253@KRBTEST.COM
14:03:17.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.792686 23898 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } has no permanent_uuid. Determining permanent_uuid...
14:03:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:17 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111397, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.252@KRBTEST.COM for kudu/127.23.16.252@KRBTEST.COM
14:03:17.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:17 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111390, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.16.254@KRBTEST.COM
14:03:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.824906 23619 external_mini_cluster.cc:955] 0 TS(s) registered with all masters
14:03:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.824920 23898 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: Bootstrap starting.
14:03:17.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.832029 23898 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: Neither blocks nor log segments found. Creating new log.
14:03:17.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.834183 23898 log.cc:826] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: Log is configured to *not* fsync() on all Append() calls
14:03:17.834 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:17.835 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:03:17.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:03:17.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:03:17.838 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.16.254"
port: 35195
}
bound_http_address {
host: "127.23.16.254"
port: 33053
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.16.253"
port: 46275
}
bound_http_address {
host: "127.23.16.253"
port: 38921
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.16.252"
port: 38825
}
bound_http_address {
host: "127.23.16.252"
port: 44041
}
}
}
14:03:17.838 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:03:17.839 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:17.839 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:03:17.839 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:03:17.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.842041 23898 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: No bootstrap required, opened a new log
14:03:17.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.871691 23898 raft_consensus.cc:359] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } }
14:03:17.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.872941 23898 raft_consensus.cc:385] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:17.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.873320 23898 raft_consensus.cc:740] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db3cc88e701c4931bfed4ebff3b7965f, State: Initialized, Role: FOLLOWER
14:03:17.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.874495 23898 consensus_queue.cc:260] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } }
14:03:17.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:17 dist-test-slave-10pc krb5kdc[23630](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111397, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:03:17.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:03:17.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.877776 23910 sys_catalog.cc:455] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } } }
14:03:17.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.878568 23910 sys_catalog.cc:458] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [sys.catalog]: This master's current role is: FOLLOWER
14:03:17.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.879262 23898 sys_catalog.cc:565] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [sys.catalog]: configured and running, proceeding with master startup.
14:03:17.884 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:17.885 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5cc
14:03:17.887 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:03:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.888419 23910 raft_consensus.cc:493] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.889333 23910 raft_consensus.cc:515] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } }
14:03:17.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.892889 23910 leader_election.cc:290] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195), feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275)
14:03:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:17.920367 23921 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:17.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:17.920742 23921 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:17.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.913926 23813 master_runner.cc:187] Error getting master registration for 127.23.16.254:35195: OK, instance_id { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" instance_seqno: 1786111392860861 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:03:17.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:17.943586 23813 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:03:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:18.122479 23850 tablet.cc:2406] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:03:18.891 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.16.252:38825
14:03:18.892 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:03:18.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:18.892835 23619 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu with pid 23813
14:03:18.926 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:18.926 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.16.254:35195
14:03:18.926 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:03:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:18.927109 23619 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu with pid 23646
14:03:18.959 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:18.959 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.16.253:46275
14:03:18.959 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:03:18.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:18.959841 23619 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu with pid 23726
14:03:18.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:18.984 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.16.252:38825
14:03:18.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:18.986012 23619 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/wal
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/logs
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data/info.pb
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:18.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.16.252:38825
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.16.252
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44041
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.16.252
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.16.212:35199
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.16.254:35195,127.23.16.253:46275,127.23.16.252:38825
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45369/jwks.json
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:03:18.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:03:19.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:19.330947 23925 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:03:19.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:19.331732 23925 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:19.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:19.332422 23925 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:03:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:19.368631 23925 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:03:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:19.368872 23925 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:03:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:19.369275 23925 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:03:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:19.369515 23925 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:03:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:19.369722 23925 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:03:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:19.369918 23925 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:03:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:19.370069 23925 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45369/jwks.json
14:03:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:19.412398 23925 master_runner.cc:387] Master server non-default flags:
14:03:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/logs
14:03:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.16.212:35199
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/wal
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.16.254:35195,127.23.16.253:46275,127.23.16.252:38825
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.16.252
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.16.252:38825
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45369/jwks.json
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data/info.pb
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.16.252
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44041
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cb747ff0ca0209e3fa98be4167c67edd474acd67
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Aug 2026 13:43:25 UTC on e7f111948823
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13227
14:03:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:03:19.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:19.413862 23925 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:19.414991 23925 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:03:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:19.427503 23931 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:19.429327 23932 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:20.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:20.741451 23925 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.315s user 0.540s sys 0.766s
14:03:20.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:20.741843 23925 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.315s user 0.540s sys 0.766s
14:03:20.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.745528 23925 server_base.cc:1061] running on GCE node
14:03:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:20.747182 23936 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:20.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.748813 23925 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:20.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.752292 23925 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:20.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.753818 23925 hybrid_clock.cc:648] HybridClock initialized: now 1786111400753702 us; error 101 us; skew 500 ppm
14:03:20.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.755074 23925 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:20.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:20 dist-test-slave-10pc krb5kdc[23630](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111400, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:03:20.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.765069 23925 init.cc:377] Logged in from keytab as kudu/127.23.16.252@KRBTEST.COM (short username kudu)
14:03:20.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.771517 23925 webserver.cc:533] Webserver started at http://127.23.16.252:44041/ using document root <none> and password file <none>
14:03:20.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.772913 23925 fs_manager.cc:362] Metadata directory not provided
14:03:20.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.773267 23925 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:20.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.786168 23925 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.008s sys 0.000s
14:03:20.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.792582 23942 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:20.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.794087 23925 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
14:03:20.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.794413 23925 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data,/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/wal
14:03:20.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db3cc88e701c4931bfed4ebff3b7965f"
14:03:20.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:17 on dist-test-slave-10pc"
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.796703 23925 fs_report.cc:389] FS layout report
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/wal
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/wal
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data/data
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:20.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:20.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.886011 23925 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:20.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.892241 23925 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:20.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.892781 23925 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:20.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.974695 23925 rpc_server.cc:307] RPC server started. Bound to: 127.23.16.252:38825
14:03:20.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.974750 23993 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.16.252:38825 every 8 connection(s)
14:03:20.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.977902 23925 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-2/data/info.pb
14:03:20.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.981446 23619 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu as pid 23925
14:03:20.982 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:20.982 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.16.254:35195
14:03:20.982 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:03:20.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.984360 23619 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:20.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:20.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal
14:03:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data
14:03:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/logs
14:03:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data/info.pb
14:03:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.16.254:35195
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.16.254
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33053
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.16.254
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.16.212:35199
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.16.254:35195,127.23.16.253:46275,127.23.16.252:38825
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45369/jwks.json
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:03:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:03:20.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:20.995906 23994 sys_catalog.cc:263] Verifying existing consensus state
14:03:21.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.001555 23994 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: Bootstrap starting.
14:03:21.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.037909 23994 log.cc:826] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: Log is configured to *not* fsync() on all Append() calls
14:03:21.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.043011 23994 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:03:21.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.043779 23994 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: Bootstrap complete.
14:03:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.067955 23994 raft_consensus.cc:359] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } }
14:03:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.071484 23994 raft_consensus.cc:385] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:03:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.071823 23994 raft_consensus.cc:740] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db3cc88e701c4931bfed4ebff3b7965f, State: Initialized, Role: FOLLOWER
14:03:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.072945 23994 consensus_queue.cc:260] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } }
14:03:21.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.076579 23999 sys_catalog.cc:455] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } } }
14:03:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.078331 23999 sys_catalog.cc:458] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [sys.catalog]: This master's current role is: FOLLOWER
14:03:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.078368 23994 sys_catalog.cc:565] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [sys.catalog]: configured and running, proceeding with master startup.
14:03:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.109045 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.109664 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:21.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.106040 23925 master_runner.cc:187] Error getting master registration for 127.23.16.254:35195: Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111),
14:03:21.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.126348 23925 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:03:21.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.148567 23948 tablet.cc:2406] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:03:21.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.226828 23999 raft_consensus.cc:493] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:21.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.227471 23999 raft_consensus.cc:515] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } }
14:03:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.231021 23999 leader_election.cc:290] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195), feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275)
14:03:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.235775 23946 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111)
14:03:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.242311 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195): Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111)
14:03:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.243093 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275): Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.243489 23946 leader_election.cc:304] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db3cc88e701c4931bfed4ebff3b7965f; no voters: 444404fe730945b1a70ccfef9b90f390, feb1147a9db74cca933cbaf1af70b146
14:03:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.244397 23999 raft_consensus.cc:2749] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:03:21.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.302220 23112 debug-util.cc:398] Leaking SignalData structure 0x7b080008b3a0 after lost signal to thread 23040
14:03:21.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.302990 23112 debug-util.cc:398] Leaking SignalData structure 0x7b080008b4e0 after lost signal to thread 23115
14:03:21.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.370635 23996 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:03:21.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.371404 23996 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:21.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.372249 23996 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:03:21.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.440548 23996 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:03:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.440920 23996 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:03:21.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.441471 23996 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:03:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.441875 23996 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:03:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.442272 23996 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:03:21.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.442653 23996 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:03:21.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.442955 23996 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45369/jwks.json
14:03:21.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.526376 23996 master_runner.cc:387] Master server non-default flags:
14:03:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/logs
14:03:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.16.212:35199
14:03:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data
14:03:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal
14:03:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:03:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:21.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.16.254:35195,127.23.16.253:46275,127.23.16.252:38825
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.16.254
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.16.254:35195
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45369/jwks.json
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data/info.pb
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.16.254
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33053
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cb747ff0ca0209e3fa98be4167c67edd474acd67
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Aug 2026 13:43:25 UTC on e7f111948823
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13227
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:03:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.528317 23996 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:21.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:21.529963 23996 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:03:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.545603 24021 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:22.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:22.114976 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:22.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:22.115540 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:22.993216 24025 raft_consensus.cc:493] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:22.993899 24025 raft_consensus.cc:515] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } }
14:03:23.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:23.018013 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195): Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111)
14:03:23.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.019625 24025 leader_election.cc:290] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195), feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275)
14:03:23.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:23.040306 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275): Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.040933 23946 leader_election.cc:304] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db3cc88e701c4931bfed4ebff3b7965f; no voters: 444404fe730945b1a70ccfef9b90f390, feb1147a9db74cca933cbaf1af70b146
14:03:23.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.041873 24025 raft_consensus.cc:2749] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:03:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:23.120165 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:23.121369 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:22.946992 24020 debug-util.cc:398] Leaking SignalData structure 0x7b0800028cc0 after lost signal to thread 23996
14:03:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:23.505522 24020 kernel_stack_watchdog.cc:198] Thread 23996 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
14:03:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:03:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:03:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:03:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:03:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:23.505769 23996 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.958s user 0.704s sys 1.140s
14:03:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:23.506168 23996 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.958s user 0.704s sys 1.140s
14:03:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:21.549954 24022 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:23.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.517138 23996 server_base.cc:1061] running on GCE node
14:03:23.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:23.517277 24030 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:23.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.518909 23996 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.522351 23996 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.523795 23996 hybrid_clock.cc:648] HybridClock initialized: now 1786111403523645 us; error 150 us; skew 500 ppm
14:03:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.525082 23996 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:23.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:23 dist-test-slave-10pc krb5kdc[23630](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111403, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:03:23.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.533190 23996 init.cc:377] Logged in from keytab as kudu/127.23.16.254@KRBTEST.COM (short username kudu)
14:03:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.544777 23996 webserver.cc:533] Webserver started at http://127.23.16.254:33053/ using document root <none> and password file <none>
14:03:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.546088 23996 fs_manager.cc:362] Metadata directory not provided
14:03:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.546402 23996 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:23.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.558990 23996 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.004s sys 0.004s
14:03:23.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.565227 24036 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.566820 23996 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
14:03:23.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.567269 23996 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data,/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal
14:03:23.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "444404fe730945b1a70ccfef9b90f390"
14:03:23.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:12 on dist-test-slave-10pc"
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.570405 23996 fs_report.cc:389] FS layout report
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data/data
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:23.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.650315 23996 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.666445 23996 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:23.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.667083 23996 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:23.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.797942 23996 rpc_server.cc:307] RPC server started. Bound to: 127.23.16.254:35195
14:03:23.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.798025 24090 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.16.254:35195 every 8 connection(s)
14:03:23.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.802381 23996 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/data/info.pb
14:03:23.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.813030 23619 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu as pid 23996
14:03:23.814 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:23.814 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.16.253:46275
14:03:23.814 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:03:23.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.816023 23619 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/logs
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data/info.pb
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.16.253:46275
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.16.253
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38921
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.16.253
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.16.212:35199
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.16.254:35195,127.23.16.253:46275,127.23.16.252:38825
14:03:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45369/jwks.json
14:03:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:03:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:03:23.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.822337 24091 sys_catalog.cc:263] Verifying existing consensus state
14:03:23.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260807 14:03:23.823828 24091 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:03:23.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.824280 23996 master.cc:562] Master@127.23.16.254:35195 shutting down...
14:03:23.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:23.953470 23996 master.cc:584] Master@127.23.16.254:35195 shutdown complete.
14:03:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:23.961742 23996 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
14:03:23.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:03:24.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.126973 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:24.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.128265 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:24.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.316102 24093 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:03:24.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.316787 24093 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:03:24.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.317538 24093 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:03:24.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.368600 24093 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:03:24.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.368932 24093 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:03:24.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.369418 24093 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:03:24.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.369769 24093 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:03:24.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.370101 24093 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:03:24.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.370417 24093 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:03:24.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.370687 24093 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:45369/jwks.json
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:24.412971 24093 master_runner.cc:387] Master server non-default flags:
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/logs
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.16.212:35199
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.16.254:35195,127.23.16.253:46275,127.23.16.252:38825
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/kudu.keytab
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.23.16.253
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.16.253:46275
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45369/jwks.json
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data/info.pb
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.16.253
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38921
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision cb747ff0ca0209e3fa98be4167c67edd474acd67
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 Aug 2026 13:43:25 UTC on e7f111948823
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13227
14:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:03:24.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:24.414681 24093 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:03:24.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:24.415820 24093 file_cache.cc:504] Constructed file cache file cache with capacity 419430
14:03:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.427593 24100 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:25.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:25.137598 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:25.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:25.138232 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:25.806365 24107 raft_consensus.cc:493] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:25.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:25.808903 24107 raft_consensus.cc:515] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } }
14:03:25.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:25.821539 24107 leader_election.cc:290] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195), feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275)
14:03:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:25.885877 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195): Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111)
14:03:25.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:25.888388 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275): Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:25.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:25.890133 23946 leader_election.cc:304] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db3cc88e701c4931bfed4ebff3b7965f; no voters: 444404fe730945b1a70ccfef9b90f390, feb1147a9db74cca933cbaf1af70b146
14:03:25.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:25.893009 24107 raft_consensus.cc:2749] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:03:26.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:26.133736 23925 master_runner.cc:187] Error getting master registration for 127.23.16.254:35195: Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111),
14:03:26.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:26.134832 23925 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:03:26.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:26.166664 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:26.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:26.169339 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:27.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:24.429315 24101 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:27.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:25.865413 24099 debug-util.cc:398] Leaking SignalData structure 0x7b0800028cc0 after lost signal to thread 24093
14:03:27.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:27.060940 24099 kernel_stack_watchdog.cc:198] Thread 24093 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 421ms:
14:03:27.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:03:27.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:03:27.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:03:27.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:03:27.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:27.061331 24102 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2632 milliseconds
14:03:27.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:27.061740 24093 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.634s user 0.750s sys 1.156s
14:03:27.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:27.062168 24093 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.634s user 0.750s sys 1.156s
14:03:27.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:27.062840 24103 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:03:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.064251 24093 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:03:27.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.066411 24093 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:03:27.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.068941 24093 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:03:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.070310 24093 hybrid_clock.cc:648] HybridClock initialized: now 1786111407070277 us; error 36 us; skew 500 ppm
14:03:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.071172 24093 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:03:27.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:27 dist-test-slave-10pc krb5kdc[23630](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111407, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:03:27.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.077539 24093 init.cc:377] Logged in from keytab as kudu/127.23.16.253@KRBTEST.COM (short username kudu)
14:03:27.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.082326 24093 webserver.cc:533] Webserver started at http://127.23.16.253:38921/ using document root <none> and password file <none>
14:03:27.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.083251 24093 fs_manager.cc:362] Metadata directory not provided
14:03:27.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.083509 24093 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:03:27.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.092263 24093 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
14:03:27.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.096940 24119 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:03:27.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.098102 24093 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
14:03:27.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.098493 24093 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data,/tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal
14:03:27.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "feb1147a9db74cca933cbaf1af70b146"
14:03:27.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 14:03:14 on dist-test-slave-10pc"
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.100538 24093 fs_report.cc:389] FS layout report
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data/data
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:03:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:03:27.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.150717 24093 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:03:27.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.156646 24093 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:03:27.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.157174 24093 kserver.cc:163] Server-wide thread pool size limit: 3276
14:03:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:27.173023 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:27.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:27.173501 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:27.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.236586 24093 rpc_server.cc:307] RPC server started. Bound to: 127.23.16.253:46275
14:03:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.236683 24172 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.16.253:46275 every 8 connection(s)
14:03:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.239583 24093 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/data/info.pb
14:03:27.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.244933 23619 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu as pid 24093
14:03:27.245 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:03:27.246 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.16.252:38825(127.23.16.252:38825)] connecting to peer
14:03:27.248 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:27.248 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.23.16.252:38825(127.23.16.252:38825)] Successfully connected to peer
14:03:27.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.251245 24173 sys_catalog.cc:263] Verifying existing consensus state
14:03:27.252 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:27.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260807 14:03:27.252375 24173 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:03:27.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.253031 24093 master.cc:562] Master@127.23.16.253:46275 shutting down...
14:03:27.255 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.256 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.256 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.257 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
14:03:27.257 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.23.16.253:46275(127.23.16.253:46275)] Successfully connected to peer
14:03:27.257 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5cc
14:03:27.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:27 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1786111397, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:03:27.266 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:03:27.267 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
14:03:27.268 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.23.16.252
14:03:27.275 [DEBUG - kudu-nio-2] (SecurityContext.java:222) Refreshing Kerberos credentials...
14:03:27.282 [DEBUG - kudu-nio-2] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/krb5kdc/krb5cc
14:03:27.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:27 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1786111397, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:03:27.288 [DEBUG - kudu-nio-2] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:03:27.288 [INFO - kudu-nio-2] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
14:03:27.289 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.23.16.253
14:03:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:27 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1786111397, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.16.252@KRBTEST.COM
14:03:27.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:27 dist-test-slave-10pc krb5kdc[23630](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1786111397, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.23.16.253@KRBTEST.COM
14:03:27.316 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x87094025, L:/127.0.0.1:59754 - R:/127.23.16.252:38825] using SASL/GSSAPI
14:03:27.318 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x349c3892, L:/127.0.0.1:35244 - R:/127.23.16.253:46275] using SASL/GSSAPI
14:03:27.336 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Service unavailable: service kudu.master.MasterService not registered on Master
14:03:27.336 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:27.337 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.23.16.253:46275(127.23.16.253:46275)] handling channelInactive
14:03:27.337 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state READY due to: connection closed
14:03:27.347 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:27.355 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:27.356 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.356 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.357 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.362 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.362 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.362 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.362 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:27.367 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:27.374 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:27.375 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.375 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.375 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.378 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.378 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.378 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.378 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:27.387 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:27.391 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:27.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.392 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.393 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.393 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.393 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.394 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:27.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:27.400529 23603 debug-util.cc:398] Leaking SignalData structure 0x7b08000bada0 after lost signal to thread 23472
14:03:27.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:27.401782 23603 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad5c0 after lost signal to thread 23606
14:03:27.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:27.404175 24093 master.cc:584] Master@127.23.16.253:46275 shutdown complete.
14:03:27.407 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:27.407 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:27.407676 24093 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
14:03:27.408 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.408 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.409 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.413 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:27.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_tJAxU/test-tmp/mini-kudu-cluster5525516116610559721/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:03:27.447 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:27.447 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:27.448 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.448 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.448 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.450 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.450 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.450 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.454 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:27.486 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:27.487 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:27.488 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.488 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.488 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.489 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.489 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.489 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.491 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:27.607 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:27.608 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:27.608 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.608 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.608 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.612 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:27.687 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:27.689 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:27.690 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.690 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.690 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.691 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.691 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.691 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.691 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:27.767 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:27.768 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:27.768 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.768 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.768 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:27.769 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.769 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.769 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:27.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:28.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:28.176651 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:28.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:28.176952 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:28.587 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:28.588 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:28.588 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:28.588 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:28.589 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:28.589 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:28.589 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:28.589 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:28.591 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:29.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:29.179934 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:29.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:29.180254 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:30.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:30.184182 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:30.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:30.184707 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:30.587 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:30.588 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:30.588 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:30.589 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:30.589 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:30.590 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:30.590 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:30.591 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:30.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:30.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:30.766973 24193 raft_consensus.cc:493] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:30.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:30.767741 24193 raft_consensus.cc:515] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } }
14:03:30.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:30 dist-test-slave-10pc krb5kdc[23630](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111410, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:03:30.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:30.770357 24193 leader_election.cc:290] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195), feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275)
14:03:30.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:30.770620 23938 init.cc:295] Successfully reacquired a new kerberos TGT
14:03:30.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:30.773113 23946 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111) [suppressed 5 similar messages]
14:03:30.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:30.776697 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195): Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111)
14:03:30.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:30.777431 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275): Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:30.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:30.777719 23946 leader_election.cc:304] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db3cc88e701c4931bfed4ebff3b7965f; no voters: 444404fe730945b1a70ccfef9b90f390, feb1147a9db74cca933cbaf1af70b146
14:03:30.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:30.778250 24193 raft_consensus.cc:2749] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:03:31.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:31.137743 23925 master_runner.cc:187] Error getting master registration for 127.23.16.254:35195: Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111),
14:03:31.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:31.138208 23925 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:03:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:31.187839 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:31.188119 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:31.607 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:31.608 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:31.608 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:31.608 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:31.608 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:31.609 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:31.609 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:31.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:31.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:32.191104 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:32.191390 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:33.147 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:33.147 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:33.148 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:33.149 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:33.149 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:33.149 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:33.149 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:33.149 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:33.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:33.194242 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:33.194515 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:34.197391 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:34.197654 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:35.200498 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:35.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:35.200769 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:35.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:35.653868 24201 raft_consensus.cc:493] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:35.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:35.654332 24201 raft_consensus.cc:515] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } }
14:03:35.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:35.656268 24201 leader_election.cc:290] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195), feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275)
14:03:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:35.662415 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195): Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111)
14:03:35.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:35.663125 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275): Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:35.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:35.663468 23946 leader_election.cc:304] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db3cc88e701c4931bfed4ebff3b7965f; no voters: 444404fe730945b1a70ccfef9b90f390, feb1147a9db74cca933cbaf1af70b146
14:03:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:35.664134 24201 raft_consensus.cc:2749] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:03:36.127 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:36.128 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:36.129 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:36.129 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:36.129 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:36.129 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:36.129 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:36.129 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:36.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:36.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:36.140502 23925 master_runner.cc:187] Error getting master registration for 127.23.16.254:35195: Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111),
14:03:36.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:36.140908 23925 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:36.203805 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:36.204073 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:37.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:37.207099 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:37.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:37.207366 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:38.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:38.164736 23603 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba5a0 after lost signal to thread 23472
14:03:38.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:38.165603 23603 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba6a0 after lost signal to thread 23606
14:03:38.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:38.210367 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:38.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:38.210731 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:39.213794 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:39.214107 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:39.567 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:39.568 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:39.568 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:39.569 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:39.569 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:39.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:39.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:39.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:39.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:40.087 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:40.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:40.089 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:40.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:40.090 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:40.090 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:40.090 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:40.091 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:40.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:40.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:40.217563 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:40.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:40.217963 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:40 dist-test-slave-10pc krb5kdc[23630](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111420, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:03:40.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:40.773348 23938 init.cc:295] Successfully reacquired a new kerberos TGT
14:03:41.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:41.145435 23925 master_runner.cc:187] Error getting master registration for 127.23.16.254:35195: Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111),
14:03:41.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:41.145946 23925 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:03:41.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:41.221329 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:41.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:41.221652 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:42.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:42.224749 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:42.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:42.225070 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:42.947 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:42.947 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:42.948 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:42.948 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:42.948 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:42.948 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:42.948 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:42.949 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:42.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:43.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:43.227996 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:43.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:43.228313 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:43.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:43.276986 22952 debug-util.cc:398] Leaking SignalData structure 0x7b0800081ac0 after lost signal to thread 22881
14:03:43.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:43.277581 22952 debug-util.cc:398] Leaking SignalData structure 0x7b080009e220 after lost signal to thread 22955
14:03:43.687 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:43.689 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:43.689 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:43.689 [INFO - pool-44-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:43.689 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:43.691 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:43.691 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:43.691 [INFO - pool-44-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:43.691 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:44.107 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:44.108 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:44.109 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:44.109 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:44.109 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:44.111 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:44.111 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:44.111 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:44.111 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:44.232838 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:44.233189 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:44.468 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:44.468 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:44.469 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:44.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:44.469 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:44.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:44.470 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:44.470 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:44.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:44.847 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:44.848 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:44.848 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:44.848 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:44.848 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:44.849 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:44.849 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:44.849 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:44.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:45.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:45.236508 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:45.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:45.236807 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:46.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:46.149269 23925 master_runner.cc:187] Error getting master registration for 127.23.16.254:35195: Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111),
14:03:46.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:46.149735 23925 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:03:46.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:46.241717 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:46.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:46.242090 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:46.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:46.980792 24221 raft_consensus.cc:493] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:46.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:46.981439 24221 raft_consensus.cc:515] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } }
14:03:46.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:46.983800 24221 leader_election.cc:290] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195), feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275)
14:03:46.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:46.988224 23946 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111) [suppressed 3 similar messages]
14:03:46.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:46.992661 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195): Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111)
14:03:46.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:46.993363 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275): Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:46.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:46.993745 23946 leader_election.cc:304] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db3cc88e701c4931bfed4ebff3b7965f; no voters: 444404fe730945b1a70ccfef9b90f390, feb1147a9db74cca933cbaf1af70b146
14:03:46.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:46.994464 24221 raft_consensus.cc:2749] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:03:47.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:47.245524 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:47.245813 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:47.487 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:47.488 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:47.489 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:47.489 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:47.489 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:47.489 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:47.489 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:47.489 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:47.491 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:48.249403 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:48.249686 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:49.187 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:49.187 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:49.188 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:49.188 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:49.188 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:49.188 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:49.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:49.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:49.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:49.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:49.252774 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:49.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:49.253057 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:50.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:50.256719 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:50.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:50.257072 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:50.527 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:50.528 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:50.528 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:50.528 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:50.528 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:50.528 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:50.528 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:50.529 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:50.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:50.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 14:03:50 dist-test-slave-10pc krb5kdc[23630](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786111430, etypes {rep=17 tkt=17 ses=17}, kudu/127.23.16.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:03:50.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:50.776062 23938 init.cc:295] Successfully reacquired a new kerberos TGT
14:03:50.826 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:50.827 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:50.828 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:50.828 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:50.828 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:50.828 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:50.828 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:50.829 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:50.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:51.154364 23925 master_runner.cc:187] Error getting master registration for 127.23.16.254:35195: Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111),
14:03:51.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:51.154949 23925 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
14:03:51.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:51.259593 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:51.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:51.259887 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:51.768 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:51.768 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:51.769 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:51.769 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:51.769 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:51.769 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:51.769 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:51.769 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:51.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:52.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:52.262885 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:52.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:52.263254 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:52.284252 24231 raft_consensus.cc:493] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:52.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:52.284719 24231 raft_consensus.cc:515] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } }
14:03:52.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:52.286586 24231 leader_election.cc:290] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195), feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275)
14:03:52.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:52.289409 23946 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111) [suppressed 1 similar messages]
14:03:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:52.292817 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195): Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111)
14:03:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:52.293649 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275): Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:52.293906 23946 leader_election.cc:304] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db3cc88e701c4931bfed4ebff3b7965f; no voters: 444404fe730945b1a70ccfef9b90f390, feb1147a9db74cca933cbaf1af70b146
14:03:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:52.294546 24231 raft_consensus.cc:2749] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:03:52.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:52.643410 23456 debug-util.cc:398] Leaking SignalData structure 0x7b08000bab20 after lost signal to thread 23327
14:03:52.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:52.644415 23456 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8920 after lost signal to thread 23459
14:03:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:53.266263 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:53.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:53.266700 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:53.927 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:53.929 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:53.929 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:53.929 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:53.929 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:53.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:53.930 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:53.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:53.931 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:54.269371 24011 catalog_manager.cc:1594] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:03:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:54.269621 24011 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:03:54.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:54.724218 24238 raft_consensus.cc:493] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:03:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:54.724790 24238 raft_consensus.cc:515] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "444404fe730945b1a70ccfef9b90f390" member_type: VOTER last_known_addr { host: "127.23.16.254" port: 35195 } } peers { permanent_uuid: "feb1147a9db74cca933cbaf1af70b146" member_type: VOTER last_known_addr { host: "127.23.16.253" port: 46275 } } peers { permanent_uuid: "db3cc88e701c4931bfed4ebff3b7965f" member_type: VOTER last_known_addr { host: "127.23.16.252" port: 38825 } }
14:03:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:54.726755 24238 leader_election.cc:290] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195), feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275)
14:03:54.727 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.254:35195(127.23.16.254:35195)] connecting to peer
14:03:54.728 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.23.16.253:46275(127.23.16.253:46275)] connecting to peer
14:03:54.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:54.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.16.254:35195(127.23.16.254:35195)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:54.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.254:35195: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195
14:03:54.729 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:54.729 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.23.16.253:46275(127.23.16.253:46275)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:54.729 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.23.16.253:46275: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
14:03:54.732 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275; will retry
14:03:54.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:54.736085 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 444404fe730945b1a70ccfef9b90f390 (127.23.16.254:35195): Network error: Client connection negotiation failed: client connection to 127.23.16.254:35195: connect: Connection refused (error 111)
14:03:54.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:54.737170 23946 leader_election.cc:336] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer feb1147a9db74cca933cbaf1af70b146 (127.23.16.253:46275): Network error: Client connection negotiation failed: client connection to 127.23.16.253:46275: connect: Connection refused (error 111)
14:03:54.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:54.737566 23946 leader_election.cc:304] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: db3cc88e701c4931bfed4ebff3b7965f; no voters: 444404fe730945b1a70ccfef9b90f390, feb1147a9db74cca933cbaf1af70b146
14:03:54.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:54.738346 24238 raft_consensus.cc:2749] T 00000000000000000000000000000000 P db3cc88e701c4931bfed4ebff3b7965f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:03:54.740 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=27487), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [10ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [75ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [89ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [90ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [104ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [111ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [116ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [116ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [129ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [132ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [132ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [146ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [147ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [147ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [148ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [167ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [167ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [204ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [207ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [208ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [244ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [245ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [360ms] refreshing cache from master, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [362ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [363ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [365ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [366ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [440ms] refreshing cache from master, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [443ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [444ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [444ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [445ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [445ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [520ms] refreshing cache from master, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [522ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [523ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [525ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [525ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [1340ms] refreshing cache from master, [1341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [1341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [1342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [1343ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [1345ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [3340ms] refreshing cache from master, [3340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [3346ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [3346ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [4360ms] refreshing cache from master, [4361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [4361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [4362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [4363ms] too many traces: truncated at 100 traces, deferred=Deferred@2002565138(state=PENDING, result=null, callback=(continuation of Deferred@1840829774 after retry RPC after error@849448990) -> (continuation of Deferred@361177547 after retry RPC after error@118069057) -> (continuation of Deferred@741351499 after retry RPC after error@313484467) -> (continuation of Deferred@1040877140 after retry RPC after error@1978842443) -> (continuation of Deferred@1483598278 after retry RPC after error@823039108) -> (continuation of Deferred@1541184727 after retry RPC after error@1000630786) -> (continuation of Deferred@212899689 after retry RPC after error@1239386319) -> (continuation of Deferred@1985646343 after retry RPC after error@1857425332) -> (continuation of Deferred@1714804436 after retry RPC after error@1402577709) -> (continuation of Deferred@1897475310 after retry RPC after error@789333718) -> (continuation of Deferred@1741137557 after retry RPC after error@2035937675) -> (continuation of Deferred@1100991506 after retry RPC after error@208144385) -> (continuation of Deferred@1447158807 after retry RPC after error@1621267197) -> (continuation of Deferred@1608662439 after retry RPC after error@319163994) -> (continuation of Deferred@1245589873 after retry RPC after error@1048541105) -> (continuation of Deferred@1891531484 after retry RPC after error@1407831631) -> (continuation of Deferred@1760024826 after retry RPC after error@769714344) -> (continuation of Deferred@780865362 after retry RPC after error@864892794) -> (continuation of Deferred@1710955001 after retry RPC after error@1341397062) -> (continuation of Deferred@972255093 after retry RPC after error@1671412182) -> (continuation of Deferred@1578520717 after retry RPC after error@827282721) -> (continuation of Deferred@325389498 after retry RPC after error@229266953) -> (continuation of Deferred@1632955657 after retry RPC after error@712591967) -> (continuation of Deferred@289575798 after retry RPC after error@1317508425) -> (continuation of Deferred@150803765 after retry RPC after error@1748409799) -> (continuation of Deferred@1844079252 after retry RPC after error@523605949) -> (continuation of Deferred@2130800083 after retry RPC after error@587534161) -> (continuation of Deferred@495288175 after retry RPC after error@775850781), errback=(continuation of Deferred@1840829774 after retry RPC after error@849448990) -> (continuation of Deferred@361177547 after retry RPC after error@118069057) -> (continuation of Deferred@741351499 after retry RPC after error@313484467) -> (continuation of Deferred@1040877140 after retry RPC after error@1978842443) -> (continuation of Deferred@1483598278 after retry RPC after error@823039108) -> (continuation of Deferred@1541184727 after retry RPC after error@1000630786) -> (continuation of Deferred@212899689 after retry RPC after error@1239386319) -> (continuation of Deferred@1985646343 after retry RPC after error@1857425332) -> (continuation of Deferred@1714804436 after retry RPC after error@1402577709) -> (continuation of Deferred@1897475310 after retry RPC after error@789333718) -> (continuation of Deferred@1741137557 after retry RPC after error@2035937675) -> (continuation of Deferred@1100991506 after retry RPC after error@208144385) -> (continuation of Deferred@1447158807 after retry RPC after error@1621267197) -> (continuation of Deferred@1608662439 after retry RPC after error@319163994) -> (continuation of Deferred@1245589873 after retry RPC after error@1048541105) -> (continuation of Deferred@1891531484 after retry RPC after error@1407831631) -> (continuation of Deferred@1760024826 after retry RPC after error@769714344) -> (continuation of Deferred@780865362 after retry RPC after error@864892794) -> (continuation of Deferred@1710955001 after retry RPC after error@1341397062) -> (continuation of Deferred@972255093 after retry RPC after error@1671412182) -> (continuation of Deferred@1578520717 after retry RPC after error@827282721) -> (continuation of Deferred@325389498 after retry RPC after error@229266953) -> (continuation of Deferred@1632955657 after retry RPC after error@712591967) -> (continuation of Deferred@289575798 after retry RPC after error@1317508425) -> (continuation of Deferred@150803765 after retry RPC after error@1748409799) -> (continuation of Deferred@1844079252 after retry RPC after error@523605949) -> (continuation of Deferred@2130800083 after retry RPC after error@587534161) -> (continuation of Deferred@495288175 after retry RPC after error@775850781)))
14:03:54.743 [DEBUG - main] (Connection.java:575) [peer master-127.23.16.252:38825(127.23.16.252:38825)] disconnecting while in state READY
14:03:54.744 [DEBUG - main] (Connection.java:575) [peer master-127.23.16.253:46275(127.23.16.253:46275)] disconnecting while in state TERMINATED
14:03:54.744 [DEBUG - main] (Connection.java:575) [peer master-127.23.16.254:35195(127.23.16.254:35195)] disconnecting while in state TERMINATED
14:03:54.744 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.23.16.252:38825(127.23.16.252:38825)] handling channelInactive
14:03:54.744 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.16.252:38825(127.23.16.252:38825)] cleaning up while in state READY due to: connection closed
14:03:54.749 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:03:54.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:54.752339 23619 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu with pid 23925
14:03:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T14:03:56Z chronyd exiting
14:03:57.055 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:03:57.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:57.057768 22868 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu with pid 23181
14:03:57.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:57.092442 22868 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu with pid 23326
14:03:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:57.126093 22868 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu with pid 23470
14:03:57.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:57.162560 22868 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu with pid 22880
14:03:57.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 14:03:57.203981 22982 connection.cc:573] client connection to 127.22.85.62:38623 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:03:57.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:57.204226 22868 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu with pid 22958
14:03:57.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 14:03:57.244432 22868 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_tJAxU/build/tsan/bin/kudu with pid 23035
14:03:58.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T14:03:58Z chronyd exiting
14:03:59.475 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=27487), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [10ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [75ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [89ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [90ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [104ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [111ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [116ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [116ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [129ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [132ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [132ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [146ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [147ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [147ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [148ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [167ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [167ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [204ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [207ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [208ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [244ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [245ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [360ms] refreshing cache from master, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [362ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [363ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [365ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [366ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [440ms] refreshing cache from master, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [443ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [444ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [444ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [445ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [445ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [520ms] refreshing cache from master, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [522ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [523ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [525ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [525ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [1340ms] refreshing cache from master, [1341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [1341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [1342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [1343ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [1345ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [3340ms] refreshing cache from master, [3340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [3346ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [3346ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [4360ms] refreshing cache from master, [4361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [4361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [4362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [4363ms] too many traces: truncated at 100 traces, deferred=Deferred@2002565138(state=PENDING, result=null, callback=(continuation of Deferred@1840829774 after retry RPC after error@849448990) -> (continuation of Deferred@361177547 after retry RPC after error@118069057) -> (continuation of Deferred@741351499 after retry RPC after error@313484467) -> (continuation of Deferred@1040877140 after retry RPC after error@1978842443) -> (continuation of Deferred@1483598278 after retry RPC after error@823039108) -> (continuation of Deferred@1541184727 after retry RPC after error@1000630786) -> (continuation of Deferred@212899689 after retry RPC after error@1239386319) -> (continuation of Deferred@1985646343 after retry RPC after error@1857425332) -> (continuation of Deferred@1714804436 after retry RPC after error@1402577709) -> (continuation of Deferred@1897475310 after retry RPC after error@789333718) -> (continuation of Deferred@1741137557 after retry RPC after error@2035937675) -> (continuation of Deferred@1100991506 after retry RPC after error@208144385) -> (continuation of Deferred@1447158807 after retry RPC after error@1621267197) -> (continuation of Deferred@1608662439 after retry RPC after error@319163994) -> (continuation of Deferred@1245589873 after retry RPC after error@1048541105) -> (continuation of Deferred@1891531484 after retry RPC after error@1407831631) -> (continuation of Deferred@1760024826 after retry RPC after error@769714344) -> (continuation of Deferred@780865362 after retry RPC after error@864892794) -> (continuation of Deferred@1710955001 after retry RPC after error@1341397062) -> (continuation of Deferred@972255093 after retry RPC after error@1671412182) -> (continuation of Deferred@1578520717 after retry RPC after error@827282721) -> (continuation of Deferred@325389498 after retry RPC after error@229266953) -> (continuation of Deferred@1632955657 after retry RPC after error@712591967) -> (continuation of Deferred@289575798 after retry RPC after error@1317508425) -> (continuation of Deferred@150803765 after retry RPC after error@1748409799) -> (continuation of Deferred@1844079252 after retry RPC after error@523605949) -> (continuation of Deferred@2130800083 after retry RPC after error@587534161) -> (continuation of Deferred@495288175 after retry RPC after error@775850781), errback=(continuation of Deferred@1840829774 after retry RPC after error@849448990) -> (continuation of Deferred@361177547 after retry RPC after error@118069057) -> (continuation of Deferred@741351499 after retry RPC after error@313484467) -> (continuation of Deferred@1040877140 after retry RPC after error@1978842443) -> (continuation of Deferred@1483598278 after retry RPC after error@823039108) -> (continuation of Deferred@1541184727 after retry RPC after error@1000630786) -> (continuation of Deferred@212899689 after retry RPC after error@1239386319) -> (continuation of Deferred@1985646343 after retry RPC after error@1857425332) -> (continuation of Deferred@1714804436 after retry RPC after error@1402577709) -> (continuation of Deferred@1897475310 after retry RPC after error@789333718) -> (continuation of Deferred@1741137557 after retry RPC after error@2035937675) -> (continuation of Deferred@1100991506 after retry RPC after error@208144385) -> (continuation of Deferred@1447158807 after retry RPC after error@1621267197) -> (continuation of Deferred@1608662439 after retry RPC after error@319163994) -> (continuation of Deferred@1245589873 after retry RPC after error@1048541105) -> (continuation of Deferred@1891531484 after retry RPC after error@1407831631) -> (continuation of Deferred@1760024826 after retry RPC after error@769714344) -> (continuation of Deferred@780865362 after retry RPC after error@864892794) -> (continuation of Deferred@1710955001 after retry RPC after error@1341397062) -> (continuation of Deferred@972255093 after retry RPC after error@1671412182) -> (continuation of Deferred@1578520717 after retry RPC after error@827282721) -> (continuation of Deferred@325389498 after retry RPC after error@229266953) -> (continuation of Deferred@1632955657 after retry RPC after error@712591967) -> (continuation of Deferred@289575798 after retry RPC after error@1317508425) -> (continuation of Deferred@150803765 after retry RPC after error@1748409799) -> (continuation of Deferred@1844079252 after retry RPC after error@523605949) -> (continuation of Deferred@2130800083 after retry RPC after error@587534161) -> (continuation of Deferred@495288175 after retry RPC after error@775850781)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:348)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:571)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:37)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:517)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.23.16.253:46275
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=27487), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [10ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [75ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [89ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [90ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [104ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [111ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [116ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [116ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [129ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [132ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [132ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [146ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [147ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [147ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [148ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [167ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [167ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [204ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [207ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [208ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [244ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [245ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [360ms] refreshing cache from master, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [362ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [363ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [365ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [366ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [440ms] refreshing cache from master, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [443ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [444ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [444ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [445ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [445ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [520ms] refreshing cache from master, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [522ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [523ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [525ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [525ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [1340ms] refreshing cache from master, [1341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [1341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [1342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [1343ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [1345ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [3340ms] refreshing cache from master, [3340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [3346ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [3346ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [4360ms] refreshing cache from master, [4361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [4361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [4362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [4363ms] too many traces: truncated at 100 traces, deferred=Deferred@2002565138(state=PENDING, result=null, callback=(continuation of Deferred@1840829774 after retry RPC after error@849448990) -> (continuation of Deferred@361177547 after retry RPC after error@118069057) -> (continuation of Deferred@741351499 after retry RPC after error@313484467) -> (continuation of Deferred@1040877140 after retry RPC after error@1978842443) -> (continuation of Deferred@1483598278 after retry RPC after error@823039108) -> (continuation of Deferred@1541184727 after retry RPC after error@1000630786) -> (continuation of Deferred@212899689 after retry RPC after error@1239386319) -> (continuation of Deferred@1985646343 after retry RPC after error@1857425332) -> (continuation of Deferred@1714804436 after retry RPC after error@1402577709) -> (continuation of Deferred@1897475310 after retry RPC after error@789333718) -> (continuation of Deferred@1741137557 after retry RPC after error@2035937675) -> (continuation of Deferred@1100991506 after retry RPC after error@208144385) -> (continuation of Deferred@1447158807 after retry RPC after error@1621267197) -> (continuation of Deferred@1608662439 after retry RPC after error@319163994) -> (continuation of Deferred@1245589873 after retry RPC after error@1048541105) -> (continuation of Deferred@1891531484 after retry RPC after error@1407831631) -> (continuation of Deferred@1760024826 after retry RPC after error@769714344) -> (continuation of Deferred@780865362 after retry RPC after error@864892794) -> (continuation of Deferred@1710955001 after retry RPC after error@1341397062) -> (continuation of Deferred@972255093 after retry RPC after error@1671412182) -> (continuation of Deferred@1578520717 after retry RPC after error@827282721) -> (continuation of Deferred@325389498 after retry RPC after error@229266953) -> (continuation of Deferred@1632955657 after retry RPC after error@712591967) -> (continuation of Deferred@289575798 after retry RPC after error@1317508425) -> (continuation of Deferred@150803765 after retry RPC after error@1748409799) -> (continuation of Deferred@1844079252 after retry RPC after error@523605949) -> (continuation of Deferred@2130800083 after retry RPC after error@587534161) -> (continuation of Deferred@495288175 after retry RPC after error@775850781), errback=(continuation of Deferred@1840829774 after retry RPC after error@849448990) -> (continuation of Deferred@361177547 after retry RPC after error@118069057) -> (continuation of Deferred@741351499 after retry RPC after error@313484467) -> (continuation of Deferred@1040877140 after retry RPC after error@1978842443) -> (continuation of Deferred@1483598278 after retry RPC after error@823039108) -> (continuation of Deferred@1541184727 after retry RPC after error@1000630786) -> (continuation of Deferred@212899689 after retry RPC after error@1239386319) -> (continuation of Deferred@1985646343 after retry RPC after error@1857425332) -> (continuation of Deferred@1714804436 after retry RPC after error@1402577709) -> (continuation of Deferred@1897475310 after retry RPC after error@789333718) -> (continuation of Deferred@1741137557 after retry RPC after error@2035937675) -> (continuation of Deferred@1100991506 after retry RPC after error@208144385) -> (continuation of Deferred@1447158807 after retry RPC after error@1621267197) -> (continuation of Deferred@1608662439 after retry RPC after error@319163994) -> (continuation of Deferred@1245589873 after retry RPC after error@1048541105) -> (continuation of Deferred@1891531484 after retry RPC after error@1407831631) -> (continuation of Deferred@1760024826 after retry RPC after error@769714344) -> (continuation of Deferred@780865362 after retry RPC after error@864892794) -> (continuation of Deferred@1710955001 after retry RPC after error@1341397062) -> (continuation of Deferred@972255093 after retry RPC after error@1671412182) -> (continuation of Deferred@1578520717 after retry RPC after error@827282721) -> (continuation of Deferred@325389498 after retry RPC after error@229266953) -> (continuation of Deferred@1632955657 after retry RPC after error@712591967) -> (continuation of Deferred@289575798 after retry RPC after error@1317508425) -> (continuation of Deferred@150803765 after retry RPC after error@1748409799) -> (continuation of Deferred@1844079252 after retry RPC after error@523605949) -> (continuation of Deferred@2130800083 after retry RPC after error@587534161) -> (continuation of Deferred@495288175 after retry RPC after error@775850781)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:348)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:571)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:37)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:517)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.23.16.253:46275
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
14:03:59.943 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: giving up after 1 attempts
14:03:59.944 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=27487), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [10ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [75ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [89ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [90ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [104ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [111ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [116ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [116ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [129ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [132ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [132ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [146ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [147ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [147ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [148ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [167ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [167ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [204ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [207ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [208ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [244ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [245ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [360ms] refreshing cache from master, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [362ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [363ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [365ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [366ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [440ms] refreshing cache from master, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [443ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [444ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [444ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [445ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [445ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [520ms] refreshing cache from master, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [522ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [523ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [525ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [525ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [1340ms] refreshing cache from master, [1341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [1341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [1342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [1343ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [1345ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [3340ms] refreshing cache from master, [3340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [3346ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [3346ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [4360ms] refreshing cache from master, [4361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [4361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [4362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [4363ms] too many traces: truncated at 100 traces, deferred=Deferred@2002565138(state=PENDING, result=null, callback=(continuation of Deferred@1840829774 after retry RPC after error@849448990) -> (continuation of Deferred@361177547 after retry RPC after error@118069057) -> (continuation of Deferred@741351499 after retry RPC after error@313484467) -> (continuation of Deferred@1040877140 after retry RPC after error@1978842443) -> (continuation of Deferred@1483598278 after retry RPC after error@823039108) -> (continuation of Deferred@1541184727 after retry RPC after error@1000630786) -> (continuation of Deferred@212899689 after retry RPC after error@1239386319) -> (continuation of Deferred@1985646343 after retry RPC after error@1857425332) -> (continuation of Deferred@1714804436 after retry RPC after error@1402577709) -> (continuation of Deferred@1897475310 after retry RPC after error@789333718) -> (continuation of Deferred@1741137557 after retry RPC after error@2035937675) -> (continuation of Deferred@1100991506 after retry RPC after error@208144385) -> (continuation of Deferred@1447158807 after retry RPC after error@1621267197) -> (continuation of Deferred@1608662439 after retry RPC after error@319163994) -> (continuation of Deferred@1245589873 after retry RPC after error@1048541105) -> (continuation of Deferred@1891531484 after retry RPC after error@1407831631) -> (continuation of Deferred@1760024826 after retry RPC after error@769714344) -> (continuation of Deferred@780865362 after retry RPC after error@864892794) -> (continuation of Deferred@1710955001 after retry RPC after error@1341397062) -> (continuation of Deferred@972255093 after retry RPC after error@1671412182) -> (continuation of Deferred@1578520717 after retry RPC after error@827282721) -> (continuation of Deferred@325389498 after retry RPC after error@229266953) -> (continuation of Deferred@1632955657 after retry RPC after error@712591967) -> (continuation of Deferred@289575798 after retry RPC after error@1317508425) -> (continuation of Deferred@150803765 after retry RPC after error@1748409799) -> (continuation of Deferred@1844079252 after retry RPC after error@523605949) -> (continuation of Deferred@2130800083 after retry RPC after error@587534161) -> (continuation of Deferred@495288175 after retry RPC after error@775850781), errback=(continuation of Deferred@1840829774 after retry RPC after error@849448990) -> (continuation of Deferred@361177547 after retry RPC after error@118069057) -> (continuation of Deferred@741351499 after retry RPC after error@313484467) -> (continuation of Deferred@1040877140 after retry RPC after error@1978842443) -> (continuation of Deferred@1483598278 after retry RPC after error@823039108) -> (continuation of Deferred@1541184727 after retry RPC after error@1000630786) -> (continuation of Deferred@212899689 after retry RPC after error@1239386319) -> (continuation of Deferred@1985646343 after retry RPC after error@1857425332) -> (continuation of Deferred@1714804436 after retry RPC after error@1402577709) -> (continuation of Deferred@1897475310 after retry RPC after error@789333718) -> (continuation of Deferred@1741137557 after retry RPC after error@2035937675) -> (continuation of Deferred@1100991506 after retry RPC after error@208144385) -> (continuation of Deferred@1447158807 after retry RPC after error@1621267197) -> (continuation of Deferred@1608662439 after retry RPC after error@319163994) -> (continuation of Deferred@1245589873 after retry RPC after error@1048541105) -> (continuation of Deferred@1891531484 after retry RPC after error@1407831631) -> (continuation of Deferred@1760024826 after retry RPC after error@769714344) -> (continuation of Deferred@780865362 after retry RPC after error@864892794) -> (continuation of Deferred@1710955001 after retry RPC after error@1341397062) -> (continuation of Deferred@972255093 after retry RPC after error@1671412182) -> (continuation of Deferred@1578520717 after retry RPC after error@827282721) -> (continuation of Deferred@325389498 after retry RPC after error@229266953) -> (continuation of Deferred@1632955657 after retry RPC after error@712591967) -> (continuation of Deferred@289575798 after retry RPC after error@1317508425) -> (continuation of Deferred@150803765 after retry RPC after error@1748409799) -> (continuation of Deferred@1844079252 after retry RPC after error@523605949) -> (continuation of Deferred@2130800083 after retry RPC after error@587534161) -> (continuation of Deferred@495288175 after retry RPC after error@775850781)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:348)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:571)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:37)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:517)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.23.16.253:46275
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=27487), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [10ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [75ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [89ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [90ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [104ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [111ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [116ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [116ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [129ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [132ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [132ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [146ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [147ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [147ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [148ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [167ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [167ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [204ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [207ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [208ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [244ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [245ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [360ms] refreshing cache from master, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [362ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [363ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [365ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [366ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [440ms] refreshing cache from master, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [443ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [444ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [444ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [445ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [445ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [520ms] refreshing cache from master, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [522ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [523ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [525ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [525ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [1340ms] refreshing cache from master, [1341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [1341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [1342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [1343ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [1345ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [3340ms] refreshing cache from master, [3340ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.254:35195: Network error: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.253:46275: Network error: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [3346ms] Sub RPC ConnectToMaster: received response from server master-127.23.16.252:38825: OK, [3346ms] delaying RPC due to: Service unavailable: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275, [4360ms] refreshing cache from master, [4361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.252:38825, [4361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.254:35195, [4362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.16.253:46275, [4363ms] too many traces: truncated at 100 traces, deferred=Deferred@2002565138(state=PENDING, result=null, callback=(continuation of Deferred@1840829774 after retry RPC after error@849448990) -> (continuation of Deferred@361177547 after retry RPC after error@118069057) -> (continuation of Deferred@741351499 after retry RPC after error@313484467) -> (continuation of Deferred@1040877140 after retry RPC after error@1978842443) -> (continuation of Deferred@1483598278 after retry RPC after error@823039108) -> (continuation of Deferred@1541184727 after retry RPC after error@1000630786) -> (continuation of Deferred@212899689 after retry RPC after error@1239386319) -> (continuation of Deferred@1985646343 after retry RPC after error@1857425332) -> (continuation of Deferred@1714804436 after retry RPC after error@1402577709) -> (continuation of Deferred@1897475310 after retry RPC after error@789333718) -> (continuation of Deferred@1741137557 after retry RPC after error@2035937675) -> (continuation of Deferred@1100991506 after retry RPC after error@208144385) -> (continuation of Deferred@1447158807 after retry RPC after error@1621267197) -> (continuation of Deferred@1608662439 after retry RPC after error@319163994) -> (continuation of Deferred@1245589873 after retry RPC after error@1048541105) -> (continuation of Deferred@1891531484 after retry RPC after error@1407831631) -> (continuation of Deferred@1760024826 after retry RPC after error@769714344) -> (continuation of Deferred@780865362 after retry RPC after error@864892794) -> (continuation of Deferred@1710955001 after retry RPC after error@1341397062) -> (continuation of Deferred@972255093 after retry RPC after error@1671412182) -> (continuation of Deferred@1578520717 after retry RPC after error@827282721) -> (continuation of Deferred@325389498 after retry RPC after error@229266953) -> (continuation of Deferred@1632955657 after retry RPC after error@712591967) -> (continuation of Deferred@289575798 after retry RPC after error@1317508425) -> (continuation of Deferred@150803765 after retry RPC after error@1748409799) -> (continuation of Deferred@1844079252 after retry RPC after error@523605949) -> (continuation of Deferred@2130800083 after retry RPC after error@587534161) -> (continuation of Deferred@495288175 after retry RPC after error@775850781), errback=(continuation of Deferred@1840829774 after retry RPC after error@849448990) -> (continuation of Deferred@361177547 after retry RPC after error@118069057) -> (continuation of Deferred@741351499 after retry RPC after error@313484467) -> (continuation of Deferred@1040877140 after retry RPC after error@1978842443) -> (continuation of Deferred@1483598278 after retry RPC after error@823039108) -> (continuation of Deferred@1541184727 after retry RPC after error@1000630786) -> (continuation of Deferred@212899689 after retry RPC after error@1239386319) -> (continuation of Deferred@1985646343 after retry RPC after error@1857425332) -> (continuation of Deferred@1714804436 after retry RPC after error@1402577709) -> (continuation of Deferred@1897475310 after retry RPC after error@789333718) -> (continuation of Deferred@1741137557 after retry RPC after error@2035937675) -> (continuation of Deferred@1100991506 after retry RPC after error@208144385) -> (continuation of Deferred@1447158807 after retry RPC after error@1621267197) -> (continuation of Deferred@1608662439 after retry RPC after error@319163994) -> (continuation of Deferred@1245589873 after retry RPC after error@1048541105) -> (continuation of Deferred@1891531484 after retry RPC after error@1407831631) -> (continuation of Deferred@1760024826 after retry RPC after error@769714344) -> (continuation of Deferred@780865362 after retry RPC after error@864892794) -> (continuation of Deferred@1710955001 after retry RPC after error@1341397062) -> (continuation of Deferred@972255093 after retry RPC after error@1671412182) -> (continuation of Deferred@1578520717 after retry RPC after error@827282721) -> (continuation of Deferred@325389498 after retry RPC after error@229266953) -> (continuation of Deferred@1632955657 after retry RPC after error@712591967) -> (continuation of Deferred@289575798 after retry RPC after error@1317508425) -> (continuation of Deferred@150803765 after retry RPC after error@1748409799) -> (continuation of Deferred@1844079252 after retry RPC after error@523605949) -> (continuation of Deferred@2130800083 after retry RPC after error@587534161) -> (continuation of Deferred@495288175 after retry RPC after error@775850781)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:348)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.23.16.252:38825,127.23.16.254:35195,127.23.16.253:46275) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.254:35195(127.23.16.254:35195): Connection refused: /127.23.16.254:35195,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.16.253:46275(127.23.16.253:46275): Connection refused: /127.23.16.253:46275
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:571)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:37)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:517)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.23.16.253:46275
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)