02:07:48.657 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:07:48.657 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task0HCHY_/test-tmp
02:07:48.657 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:07:48.657 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task0HCHY_/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:07:48.660 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:07:49.028 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:07:49.028 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:07:49.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:49.031939 1374 mini_oidc.cc:150] Starting JWKS server
02:07:49.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:49.035561 1374 webserver.cc:533] Webserver started at http://127.0.0.1:43741/ using document root <none> and password file <none>
02:07:49.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:49.036033 1374 mini_oidc.cc:186] Starting OIDC Discovery server
02:07:49.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:49.038160 1374 webserver.cc:533] Webserver started at http://127.0.0.1:39389/ using document root <none> and password file <none>
02:07:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T02:07:49Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:07:49.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T02:07:49Z Disabled control of system clock
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:49.122457 1374 external_mini_cluster.cc:1372] Running /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/wal
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/data
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/logs
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/data/info.pb
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.87.190:33775
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.87.190
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.87.148:42747
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.87.190:33775,127.1.87.189:38973,127.1.87.188:36907
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43741/jwks.json with env {}
02:07:49.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:49.480942 1395 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:07:49.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:49.481608 1395 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:07:49.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:49.482345 1395 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:07:49.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:49.519309 1395 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:07:49.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:49.519618 1395 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:07:49.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:49.519829 1395 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:07:49.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:49.520044 1395 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:07:49.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:49.520241 1395 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:07:49.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:49.520421 1395 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43741/jwks.json
02:07:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:49.564284 1395 master_runner.cc:387] Master server non-default flags:
02:07:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/logs
02:07:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.87.148:42747
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/data
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/wal
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.87.190:33775,127.1.87.189:38973,127.1.87.188:36907
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.87.190:33775
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43741/jwks.json
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/data/info.pb
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.87.190
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d9153c248733781bf69132f2d094d9d3c92b2bc
02:07:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:07:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 01:45:55 UTC on e7f111948823
02:07:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13299
02:07:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:07:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:49.566231 1395 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:07:49.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:49.568039 1395 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:07:49.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:49.581306 1402 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:49.583829 1404 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:50.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:50.931233 1405 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1345 milliseconds
02:07:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:50.934260 1406 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:50.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:50.937029 1395 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.354s user 0.001s sys 0.008s
02:07:50.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:50.937320 1395 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.354s user 0.001s sys 0.008s
02:07:50.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.937606 1395 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:07:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.938827 1395 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:07:50.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.942162 1395 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:07:50.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.943626 1395 hybrid_clock.cc:648] HybridClock initialized: now 1786500470943523 us; error 121 us; skew 500 ppm
02:07:50.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.944661 1395 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:07:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.953882 1395 webserver.cc:533] Webserver started at http://127.1.87.190:41773/ using document root <none> and password file <none>
02:07:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.955078 1395 fs_manager.cc:362] Metadata directory not provided
02:07:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.955315 1395 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:07:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.955758 1395 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:07:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.960377 1395 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/data/instance:
02:07:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d263cd0c1e414172851bc013bcd4b982"
02:07:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:07:50 on dist-test-slave-j2vl"
02:07:50.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.961696 1395 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/wal/instance:
02:07:50.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d263cd0c1e414172851bc013bcd4b982"
02:07:50.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:07:50 on dist-test-slave-j2vl"
02:07:50.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.971735 1395 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
02:07:50.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.979001 1418 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:07:50.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.980347 1395 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
02:07:50.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.980792 1395 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/data,/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/wal
02:07:50.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d263cd0c1e414172851bc013bcd4b982"
02:07:50.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:07:50 on dist-test-slave-j2vl"
02:07:50.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:50.981319 1395 fs_report.cc:389] FS layout report
02:07:50.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/wal
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/wal
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/data/data
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:07:50.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:07:51.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.106206 1395 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:07:51.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.107923 1395 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:07:51.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.108426 1395 kserver.cc:163] Server-wide thread pool size limit: 3276
02:07:51.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.190464 1395 rpc_server.cc:307] RPC server started. Bound to: 127.1.87.190:33775
02:07:51.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.190923 1520 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.87.190:33775 every 8 connection(s)
02:07:51.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.193871 1395 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/data/info.pb
02:07:51.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.198155 1374 external_mini_cluster.cc:1434] Started /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu as pid 1395
02:07:51.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.199167 1374 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-0/wal/instance
02:07:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.203624 1521 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.205166 1374 external_mini_cluster.cc:1372] Running /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/wal
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/data
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/logs
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/data/info.pb
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:07:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.87.189:38973
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.87.189
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.87.148:42747
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.87.190:33775,127.1.87.189:38973,127.1.87.188:36907
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43741/jwks.json with env {}
02:07:51.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.220477 1521 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } has no permanent_uuid. Determining permanent_uuid...
02:07:51.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.258172 1521 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } has no permanent_uuid. Determining permanent_uuid...
02:07:51.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.262585 1423 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.87.189:38973: connect: Connection refused (error 111)
02:07:51.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.268318 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.189:38973: Network error: Client connection negotiation failed: client connection to 127.1.87.189:38973: connect: Connection refused (error 111)
02:07:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.318078 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } attempt: 1
02:07:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.323354 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.189:38973: Network error: Client connection negotiation failed: client connection to 127.1.87.189:38973: connect: Connection refused (error 111)
02:07:51.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.391902 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } attempt: 2
02:07:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.397307 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.189:38973: Network error: Client connection negotiation failed: client connection to 127.1.87.189:38973: connect: Connection refused (error 111)
02:07:51.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.488968 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } attempt: 3
02:07:51.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.493893 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.189:38973: Network error: Client connection negotiation failed: client connection to 127.1.87.189:38973: connect: Connection refused (error 111)
02:07:51.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.581163 1527 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:07:51.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.581874 1527 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:07:51.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.582540 1527 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:07:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.620854 1527 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:07:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.621287 1527 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:07:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.621630 1527 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.621919 1527 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.622139 1527 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.622371 1527 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43741/jwks.json
02:07:51.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.637483 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } attempt: 4
02:07:51.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.642609 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.189:38973: Network error: Client connection negotiation failed: client connection to 127.1.87.189:38973: connect: Connection refused (error 111)
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.664564 1527 master_runner.cc:387] Master server non-default flags:
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/logs
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.87.148:42747
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/data
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/wal
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.87.190:33775,127.1.87.189:38973,127.1.87.188:36907
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.87.189:38973
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43741/jwks.json
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/data/info.pb
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.87.189
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d9153c248733781bf69132f2d094d9d3c92b2bc
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 01:45:55 UTC on e7f111948823
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13299
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:07:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.666175 1527 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:07:51.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.668340 1527 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:07:51.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.681305 1545 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:51.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:51.942143 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } attempt: 5
02:07:51.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.950701 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.189:38973: Network error: Client connection negotiation failed: client connection to 127.1.87.189:38973: connect: Connection refused (error 111)
02:07:52.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:52.498459 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } attempt: 6
02:07:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:52.514734 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.189:38973: Network error: Client connection negotiation failed: client connection to 127.1.87.189:38973: connect: Connection refused (error 111)
02:07:52.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:51.683250 1546 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:52.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:52.967360 1552 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:52.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:52.970762 1551 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1286 milliseconds
02:07:52.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:52.970921 1527 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:07:52.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:52.972425 1527 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:07:52.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:52.975669 1527 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:07:52.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:52.977185 1527 hybrid_clock.cc:648] HybridClock initialized: now 1786500472977122 us; error 74 us; skew 500 ppm
02:07:52.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:52.978240 1527 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:07:52.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:52.989071 1527 webserver.cc:533] Webserver started at http://127.1.87.189:43483/ using document root <none> and password file <none>
02:07:52.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:52.990381 1527 fs_manager.cc:362] Metadata directory not provided
02:07:52.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:52.990666 1527 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:07:52.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:52.991250 1527 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:07:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:52.996172 1527 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/data/instance:
02:07:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff831d7332594234aece32e5ecf6f9b7"
02:07:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:07:52 on dist-test-slave-j2vl"
02:07:52.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:52.997607 1527 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/wal/instance:
02:07:52.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff831d7332594234aece32e5ecf6f9b7"
02:07:52.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:07:52 on dist-test-slave-j2vl"
02:07:53.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.007956 1527 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.009s sys 0.000s
02:07:53.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.016389 1570 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:07:53.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.017940 1527 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
02:07:53.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.018365 1527 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/data,/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/wal
02:07:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff831d7332594234aece32e5ecf6f9b7"
02:07:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:07:52 on dist-test-slave-j2vl"
02:07:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.018762 1527 fs_report.cc:389] FS layout report
02:07:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:07:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/wal
02:07:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/wal
02:07:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/data/data
02:07:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:07:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:07:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:07:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:07:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:07:53.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:07:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:07:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:07:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:07:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:07:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:07:53.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.098667 1527 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:07:53.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.100390 1527 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:07:53.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.100925 1527 kserver.cc:163] Server-wide thread pool size limit: 3276
02:07:53.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.197918 1527 rpc_server.cc:307] RPC server started. Bound to: 127.1.87.189:38973
02:07:53.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.198001 1671 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.87.189:38973 every 8 connection(s)
02:07:53.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.203150 1527 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/data/info.pb
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.209517 1374 external_mini_cluster.cc:1434] Started /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu as pid 1527
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.210062 1374 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-1/wal/instance
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.213267 1374 external_mini_cluster.cc:1372] Running /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/wal
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/data
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/logs
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/data/info.pb
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:07:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.87.188:36907
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.87.188
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.87.148:42747
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.87.190:33775,127.1.87.189:38973,127.1.87.188:36907
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43741/jwks.json with env {}
02:07:53.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.214115 1672 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:07:53.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.232841 1672 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } has no permanent_uuid. Determining permanent_uuid...
02:07:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.262530 1672 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } has no permanent_uuid. Determining permanent_uuid...
02:07:53.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.279884 1672 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } has no permanent_uuid. Determining permanent_uuid...
02:07:53.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.283355 1572 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:53.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.288460 1672 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:53.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.338109 1672 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 1
02:07:53.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.343482 1672 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.411993 1672 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 2
02:07:53.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.417282 1672 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:53.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.508875 1672 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 3
02:07:53.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.514753 1672 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:53.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.575423 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } attempt: 7
02:07:53.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.589838 1521 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } has no permanent_uuid. Determining permanent_uuid...
02:07:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.595192 1675 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:07:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.595925 1675 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:07:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.596763 1675 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:07:53.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.597275 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.633657 1675 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:07:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.633984 1675 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:07:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.634186 1675 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:07:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.634389 1675 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:07:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.634590 1675 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:07:53.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.634771 1675 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43741/jwks.json
02:07:53.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.655800 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 1
02:07:53.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.658288 1672 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 4
02:07:53.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.661849 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:53.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.663336 1672 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.679394 1675 master_runner.cc:387] Master server non-default flags:
02:07:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/logs
02:07:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.87.148:42747
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/data
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/wal
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.87.190:33775,127.1.87.189:38973,127.1.87.188:36907
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.87.188:36907
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43741/jwks.json
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/data/info.pb
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.87.188
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d9153c248733781bf69132f2d094d9d3c92b2bc
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 01:45:55 UTC on e7f111948823
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13299
02:07:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:07:53.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.680928 1675 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:07:53.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.682816 1675 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:07:53.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.695288 1700 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.743328 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 2
02:07:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.753011 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.838670 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 3
02:07:53.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.850404 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:53.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.962881 1672 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 5
02:07:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.974048 1672 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:53.991142 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 4
02:07:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:54.001526 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:54.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:54.285218 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 5
02:07:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:54.299575 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:54.521764 1672 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 6
02:07:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:54.560063 1672 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:54.852633 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 6
02:07:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:54.869256 1521 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.87.188:36907: Network error: Client connection negotiation failed: client connection to 127.1.87.188:36907: connect: Connection refused (error 111)
02:07:55.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:55.101689 1699 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee00 after lost signal to thread 1675
02:07:55.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:55.105274 1699 kernel_stack_watchdog.cc:198] Thread 1675 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
02:07:55.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:07:55.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:07:55.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:07:55.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:07:55.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:55.106999 1675 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.411s user 0.445s sys 0.957s
02:07:55.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:53.696290 1702 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:55.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:55.107717 1675 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.412s user 0.446s sys 0.958s
02:07:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:55.107898 1707 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1409 milliseconds
02:07:55.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.109458 1675 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:07:55.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:55.109512 1708 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.112917 1675 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:07:55.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.115679 1675 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:07:55.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.117126 1675 hybrid_clock.cc:648] HybridClock initialized: now 1786500475117081 us; error 51 us; skew 500 ppm
02:07:55.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.118091 1675 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:07:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.125141 1675 webserver.cc:533] Webserver started at http://127.1.87.188:34493/ using document root <none> and password file <none>
02:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.126466 1675 fs_manager.cc:362] Metadata directory not provided
02:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.126776 1675 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:07:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.127337 1675 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:07:55.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.132335 1675 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/data/instance:
02:07:55.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d151fa2fd0a440cba0a439aea691e64f"
02:07:55.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:07:55 on dist-test-slave-j2vl"
02:07:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.133852 1675 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/wal/instance:
02:07:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d151fa2fd0a440cba0a439aea691e64f"
02:07:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:07:55 on dist-test-slave-j2vl"
02:07:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.142549 1675 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.002s
02:07:55.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.149704 1731 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:07:55.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.151225 1675 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.151691 1675 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/data,/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/wal
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d151fa2fd0a440cba0a439aea691e64f"
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:07:55 on dist-test-slave-j2vl"
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.152094 1675 fs_report.cc:389] FS layout report
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/wal
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/wal
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/data/data
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:07:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:07:55.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:07:55.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:07:55.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:07:55.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.215193 1675 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:07:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.216943 1675 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:07:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.217545 1675 kserver.cc:163] Server-wide thread pool size limit: 3276
02:07:55.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.298115 1675 rpc_server.cc:307] RPC server started. Bound to: 127.1.87.188:36907
02:07:55.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.298543 1830 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.87.188:36907 every 8 connection(s)
02:07:55.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.302196 1675 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/data/info.pb
02:07:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.308347 1374 external_mini_cluster.cc:1434] Started /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu as pid 1675
02:07:55.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.308868 1374 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/master-2/wal/instance
02:07:55.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.310184 1832 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:07:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.330353 1832 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } has no permanent_uuid. Determining permanent_uuid...
02:07:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.360518 1832 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } has no permanent_uuid. Determining permanent_uuid...
02:07:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.372439 1832 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } has no permanent_uuid. Determining permanent_uuid...
02:07:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.392400 1832 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f: Bootstrap starting.
02:07:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.400053 1832 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f: Neither blocks nor log segments found. Creating new log.
02:07:55.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.403005 1832 log.cc:826] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f: Log is configured to *not* fsync() on all Append() calls
02:07:55.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.407953 1832 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f: No bootstrap required, opened a new log
02:07:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.429836 1832 raft_consensus.cc:359] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } }
02:07:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.430639 1832 raft_consensus.cc:385] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:07:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.430974 1832 raft_consensus.cc:740] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d151fa2fd0a440cba0a439aea691e64f, State: Initialized, Role: FOLLOWER
02:07:55.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.431924 1832 consensus_queue.cc:260] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } }
02:07:55.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.434432 1845 sys_catalog.cc:455] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } } }
02:07:55.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.435194 1845 sys_catalog.cc:458] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [sys.catalog]: This master's current role is: FOLLOWER
02:07:55.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.436990 1832 sys_catalog.cc:565] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [sys.catalog]: configured and running, proceeding with master startup.
02:07:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:55.466293 1870 catalog_manager.cc:1594] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:55.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:55.466956 1870 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:55.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:55.473141 1738 tablet.cc:2406] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:07:55.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.461553 1675 master_runner.cc:187] Error getting master registration for 127.1.87.190:33775: OK, instance_id { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" instance_seqno: 1786500471168108 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:07:55.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.528227 1675 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:07:55.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.620831 1672 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 7
02:07:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.642009 1672 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7: Bootstrap starting.
02:07:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.648850 1672 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7: Neither blocks nor log segments found. Creating new log.
02:07:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.651612 1672 log.cc:826] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7: Log is configured to *not* fsync() on all Append() calls
02:07:55.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.656955 1672 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7: No bootstrap required, opened a new log
02:07:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.678612 1672 raft_consensus.cc:359] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } }
02:07:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.679260 1672 raft_consensus.cc:385] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:07:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.679466 1672 raft_consensus.cc:740] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff831d7332594234aece32e5ecf6f9b7, State: Initialized, Role: FOLLOWER
02:07:55.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.680524 1672 consensus_queue.cc:260] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } }
02:07:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.683346 1877 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } } }
02:07:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.684262 1877 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [sys.catalog]: This master's current role is: FOLLOWER
02:07:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.684870 1672 sys_catalog.cc:565] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [sys.catalog]: configured and running, proceeding with master startup.
02:07:55.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:55.720036 1895 catalog_manager.cc:1594] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:55.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:55.720768 1895 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:55.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.744417 1845 raft_consensus.cc:493] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:07:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.745086 1845 raft_consensus.cc:515] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } }
02:07:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.714174 1527 master_runner.cc:187] Error getting master registration for 127.1.87.190:33775: OK, instance_id { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" instance_seqno: 1786500471168108 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:07:55.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.751806 1527 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:07:55.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.753259 1622 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d151fa2fd0a440cba0a439aea691e64f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ff831d7332594234aece32e5ecf6f9b7" is_pre_election: true
02:07:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.753603 1845 leader_election.cc:290] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d263cd0c1e414172851bc013bcd4b982 (127.1.87.190:33775), ff831d7332594234aece32e5ecf6f9b7 (127.1.87.189:38973)
02:07:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.754464 1622 raft_consensus.cc:2468] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d151fa2fd0a440cba0a439aea691e64f in term 0.
02:07:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.756786 1736 leader_election.cc:304] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d151fa2fd0a440cba0a439aea691e64f, ff831d7332594234aece32e5ecf6f9b7; no voters:
02:07:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.758630 1845 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [term 0 FOLLOWER]: Leader pre-election won for term 1
02:07:55.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.760607 1845 raft_consensus.cc:493] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:07:55.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.761811 1845 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [term 0 FOLLOWER]: Advancing to term 1
02:07:55.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.776460 1845 raft_consensus.cc:515] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } }
02:07:55.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.782821 1845 leader_election.cc:290] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [CANDIDATE]: Term 1 election: Requested vote from peers d263cd0c1e414172851bc013bcd4b982 (127.1.87.190:33775), ff831d7332594234aece32e5ecf6f9b7 (127.1.87.189:38973)
02:07:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.784662 1622 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d151fa2fd0a440cba0a439aea691e64f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ff831d7332594234aece32e5ecf6f9b7"
02:07:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.785274 1622 raft_consensus.cc:3060] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [term 0 FOLLOWER]: Advancing to term 1
02:07:55.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.794114 1622 raft_consensus.cc:2468] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d151fa2fd0a440cba0a439aea691e64f in term 1.
02:07:55.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.795542 1736 leader_election.cc:304] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d151fa2fd0a440cba0a439aea691e64f, ff831d7332594234aece32e5ecf6f9b7; no voters:
02:07:55.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.796568 1845 raft_consensus.cc:2804] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [term 1 FOLLOWER]: Leader election won for term 1
02:07:55.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.798756 1845 raft_consensus.cc:697] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [term 1 LEADER]: Becoming Leader. State: Replica: d151fa2fd0a440cba0a439aea691e64f, State: Running, Role: LEADER
02:07:55.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.799948 1845 consensus_queue.cc:237] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } }
02:07:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.808616 1898 sys_catalog.cc:455] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [sys.catalog]: SysCatalogTable state changed. Reason: New leader d151fa2fd0a440cba0a439aea691e64f. Latest consensus state: current_term: 1 leader_uuid: "d151fa2fd0a440cba0a439aea691e64f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } } }
02:07:55.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.809656 1898 sys_catalog.cc:458] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [sys.catalog]: This master's current role is: LEADER
02:07:55.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.822188 1902 catalog_manager.cc:1511] Loading table and tablet metadata into memory...
02:07:55.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.830299 1902 catalog_manager.cc:1520] Initializing Kudu cluster ID...
02:07:55.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:55.864287 1581 tablet.cc:2406] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:07:55.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.866214 1622 raft_consensus.cc:1275] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [term 1 FOLLOWER]: Refusing update from remote peer d151fa2fd0a440cba0a439aea691e64f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:07:55.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.868322 1898 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:07:55.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.905490 1521 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } attempt: 7
02:07:55.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.906638 1904 mvcc.cc:204] Tried to move back new op lower bound from 7317505949099675648 to 7317505948887830528. Current Snapshot: MvccSnapshot[applied={T|T < 7317505949099675648}]
02:07:55.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.911504 1877 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d151fa2fd0a440cba0a439aea691e64f. Latest consensus state: current_term: 1 leader_uuid: "d151fa2fd0a440cba0a439aea691e64f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } } }
02:07:55.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.913115 1877 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [sys.catalog]: This master's current role is: FOLLOWER
02:07:55.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.917701 1845 sys_catalog.cc:455] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d151fa2fd0a440cba0a439aea691e64f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } } }
02:07:55.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.918479 1845 sys_catalog.cc:458] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [sys.catalog]: This master's current role is: LEADER
02:07:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.924161 1902 catalog_manager.cc:1383] Generated new cluster ID: d96e8e33225d495b88508c4e91ab68dc
02:07:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.924487 1902 catalog_manager.cc:1531] Initializing Kudu internal certificate authority...
02:07:55.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.959220 1877 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d151fa2fd0a440cba0a439aea691e64f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } } }
02:07:55.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.960163 1877 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7 [sys.catalog]: This master's current role is: FOLLOWER
02:07:55.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.978121 1521 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982: Bootstrap starting.
02:07:55.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.986805 1902 catalog_manager.cc:1406] Generated new certificate authority record
02:07:55.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:55.988893 1902 catalog_manager.cc:1540] Loading token signing keys...
02:07:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.012909 1521 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982: Neither blocks nor log segments found. Creating new log.
02:07:56.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.015672 1521 log.cc:826] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982: Log is configured to *not* fsync() on all Append() calls
02:07:56.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.025701 1902 catalog_manager.cc:6092] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f: Generated new TSK 0
02:07:56.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.026872 1902 catalog_manager.cc:1550] Initializing in-progress tserver states...
02:07:56.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.033676 1521 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982: No bootstrap required, opened a new log
02:07:56.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.064706 1521 raft_consensus.cc:359] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } }
02:07:56.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.065799 1521 raft_consensus.cc:385] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:07:56.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.066144 1521 raft_consensus.cc:740] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d263cd0c1e414172851bc013bcd4b982, State: Initialized, Role: FOLLOWER
02:07:56.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.067159 1521 consensus_queue.cc:260] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } }
02:07:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.075580 1374 external_mini_cluster.cc:1372] Running /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:07:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:07:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/wal
02:07:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/data
02:07:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:07:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:07:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:07:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/logs
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/data/info.pb
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.87.129:0
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.87.129
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.87.129
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.87.190:33775,127.1.87.189:38973,127.1.87.188:36907
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.87.148:42747
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:07:56.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.078224 1920 sys_catalog.cc:455] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } } }
02:07:56.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.083896 1920 sys_catalog.cc:458] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [sys.catalog]: This master's current role is: FOLLOWER
02:07:56.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.089355 1463 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d151fa2fd0a440cba0a439aea691e64f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d263cd0c1e414172851bc013bcd4b982"
02:07:56.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.090306 1463 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [term 0 FOLLOWER]: Advancing to term 1
02:07:56.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.093596 1465 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d151fa2fd0a440cba0a439aea691e64f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d263cd0c1e414172851bc013bcd4b982" is_pre_election: true
02:07:56.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.101862 1463 raft_consensus.cc:2468] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d151fa2fd0a440cba0a439aea691e64f in term 1.
02:07:56.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.103183 1462 raft_consensus.cc:1275] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [term 1 FOLLOWER]: Refusing update from remote peer d151fa2fd0a440cba0a439aea691e64f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:07:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.108736 1845 consensus_queue.cc:1048] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:07:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.120131 1521 sys_catalog.cc:565] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [sys.catalog]: configured and running, proceeding with master startup.
02:07:56.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:56.121145 1431 tablet.cc:2406] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:07:56.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.180104 1935 sys_catalog.cc:455] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d151fa2fd0a440cba0a439aea691e64f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } } }
02:07:56.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.181234 1935 sys_catalog.cc:458] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [sys.catalog]: This master's current role is: FOLLOWER
02:07:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.181421 1920 sys_catalog.cc:455] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d151fa2fd0a440cba0a439aea691e64f. Latest consensus state: current_term: 1 leader_uuid: "d151fa2fd0a440cba0a439aea691e64f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } } }
02:07:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.182160 1920 sys_catalog.cc:458] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982 [sys.catalog]: This master's current role is: FOLLOWER
02:07:56.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.270972 1951 catalog_manager.cc:1295] Loaded cluster ID: d96e8e33225d495b88508c4e91ab68dc
02:07:56.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.271278 1951 catalog_manager.cc:1588] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982: loading cluster ID for follower catalog manager: success
02:07:56.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.282017 1951 catalog_manager.cc:1610] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982: acquiring CA information for follower catalog manager: success
02:07:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.305471 1951 catalog_manager.cc:1638] T 00000000000000000000000000000000 P d263cd0c1e414172851bc013bcd4b982: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:07:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:56.539646 1819 debug-util.cc:398] Leaking SignalData structure 0x7b0800070060 after lost signal to thread 1682
02:07:56.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.601792 1845 sys_catalog.cc:455] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d151fa2fd0a440cba0a439aea691e64f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d263cd0c1e414172851bc013bcd4b982" member_type: VOTER last_known_addr { host: "127.1.87.190" port: 33775 } } peers { permanent_uuid: "ff831d7332594234aece32e5ecf6f9b7" member_type: VOTER last_known_addr { host: "127.1.87.189" port: 38973 } } peers { permanent_uuid: "d151fa2fd0a440cba0a439aea691e64f" member_type: VOTER last_known_addr { host: "127.1.87.188" port: 36907 } } }
02:07:56.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.602625 1845 sys_catalog.cc:458] T 00000000000000000000000000000000 P d151fa2fd0a440cba0a439aea691e64f [sys.catalog]: This master's current role is: LEADER
02:07:56.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:56.650938 1923 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:07:56.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:56.651958 1923 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:07:56.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:56.653194 1923 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:07:56.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:56.726135 1923 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:07:56.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.727329 1895 catalog_manager.cc:1295] Loaded cluster ID: d96e8e33225d495b88508c4e91ab68dc
02:07:56.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.727706 1895 catalog_manager.cc:1588] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7: loading cluster ID for follower catalog manager: success
02:07:56.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:56.727784 1923 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.87.129
02:07:56.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.741783 1895 catalog_manager.cc:1610] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7: acquiring CA information for follower catalog manager: success
02:07:56.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.767751 1895 catalog_manager.cc:1638] T 00000000000000000000000000000000 P ff831d7332594234aece32e5ecf6f9b7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.805858 1923 tablet_server_runner.cc:78] Tablet server non-default flags:
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/logs
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.87.148:42747
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/data
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/wal
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.87.129:0
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/data/info.pb
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.87.129
02:07:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.87.190:33775,127.1.87.189:38973,127.1.87.188:36907
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.87.129
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d9153c248733781bf69132f2d094d9d3c92b2bc
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 01:45:55 UTC on e7f111948823
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13299
02:07:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:07:56.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.809056 1923 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:07:56.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:56.812331 1923 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:07:56.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:56.852873 1973 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:59.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:58.247750 1970 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 1923
02:07:59.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:59.125208 1923 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.268s user 0.472s sys 0.705s
02:07:59.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:59.133122 1923 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.276s user 0.473s sys 0.705s
02:07:59.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:56.859616 1974 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:59.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.155478 1923 server_base.cc:1061] running on GCE node
02:07:59.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:07:59.162015 1982 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:59.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.163766 1923 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:07:59.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.185634 1923 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:07:59.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.187182 1923 hybrid_clock.cc:648] HybridClock initialized: now 1786500479187151 us; error 44 us; skew 500 ppm
02:07:59.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.188489 1923 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:07:59.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.232090 1923 webserver.cc:533] Webserver started at http://127.1.87.129:42699/ using document root <none> and password file <none>
02:07:59.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.233623 1923 fs_manager.cc:362] Metadata directory not provided
02:07:59.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.234048 1923 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:07:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.234797 1923 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.262885 1923 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/data/instance:
02:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb6934b7f5bf4e6c903aa4e9ae0e173e"
02:07:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:07:59 on dist-test-slave-j2vl"
02:07:59.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.264797 1923 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/wal/instance:
02:07:59.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb6934b7f5bf4e6c903aa4e9ae0e173e"
02:07:59.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:07:59 on dist-test-slave-j2vl"
02:07:59.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.315115 1923 fs_manager.cc:696] Time spent creating directory manager: real 0.049s user 0.005s sys 0.005s
02:07:59.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.348776 2000 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:07:59.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.352537 1923 fs_manager.cc:730] Time spent opening block manager: real 0.023s user 0.001s sys 0.004s
02:07:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.353032 1923 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/data,/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/wal
02:07:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb6934b7f5bf4e6c903aa4e9ae0e173e"
02:07:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:07:59 on dist-test-slave-j2vl"
02:07:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.354051 1923 fs_report.cc:389] FS layout report
02:07:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/wal
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/wal
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/data/data
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:07:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:07:59.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.626843 1923 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:07:59.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.629482 1923 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:07:59.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.630388 1923 kserver.cc:163] Server-wide thread pool size limit: 3276
02:07:59.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.643545 1923 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:07:59.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.664660 1923 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:07:59.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.665230 1923 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.001s
02:07:59.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.665704 1923 ts_tablet_manager.cc:616] Registered 0 tablets
02:07:59.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:07:59.666064 1923 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:08:00.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.664525 2224 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.87.129:41995 every 8 connection(s)
02:08:00.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.667863 1923 rpc_server.cc:307] RPC server started. Bound to: 127.1.87.129:41995
02:08:00.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.671013 1923 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/data/info.pb
02:08:00.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.676930 1374 external_mini_cluster.cc:1434] Started /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu as pid 1923
02:08:00.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.677657 1374 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-0/wal/instance
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.696784 1374 external_mini_cluster.cc:1372] Running /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/wal
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/data
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/logs
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/data/info.pb
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.87.130:0
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.87.130
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.87.130
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.87.190:33775,127.1.87.189:38973,127.1.87.188:36907
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.87.148:42747
02:08:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:00.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:00.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:08:00.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.740072 2227 heartbeater.cc:344] Connected to a master server at 127.1.87.188:36907
02:08:00.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.740975 2227 heartbeater.cc:461] Registering TS with master...
02:08:00.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.742928 2227 heartbeater.cc:507] Master 127.1.87.188:36907 requested a full tablet report, sending...
02:08:00.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.748960 1762 ts_manager.cc:194] Registered new tserver with Master: eb6934b7f5bf4e6c903aa4e9ae0e173e (127.1.87.129:41995)
02:08:00.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.753597 1762 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.87.129:59461
02:08:00.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.775053 2228 heartbeater.cc:344] Connected to a master server at 127.1.87.189:38973
02:08:00.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.775477 2228 heartbeater.cc:461] Registering TS with master...
02:08:00.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.776188 2228 heartbeater.cc:507] Master 127.1.87.189:38973 requested a full tablet report, sending...
02:08:00.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.779093 2239 heartbeater.cc:344] Connected to a master server at 127.1.87.190:33775
02:08:00.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.779557 2239 heartbeater.cc:461] Registering TS with master...
02:08:00.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.780308 2239 heartbeater.cc:507] Master 127.1.87.190:33775 requested a full tablet report, sending...
02:08:00.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.780483 1596 ts_manager.cc:194] Registered new tserver with Master: eb6934b7f5bf4e6c903aa4e9ae0e173e (127.1.87.129:41995)
02:08:00.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:00.784271 1451 ts_manager.cc:194] Registered new tserver with Master: eb6934b7f5bf4e6c903aa4e9ae0e173e (127.1.87.129:41995)
02:08:01.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:01.197932 2244 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:01.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:01.198750 2244 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:01.199565 2244 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:01.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:01.256484 2244 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:01.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:01.257965 2244 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.87.130
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:01.303314 2244 tablet_server_runner.cc:78] Tablet server non-default flags:
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/logs
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.87.148:42747
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/data
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/wal
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.87.130:0
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/data/info.pb
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.87.130
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.87.190:33775,127.1.87.189:38973,127.1.87.188:36907
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.87.130
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d9153c248733781bf69132f2d094d9d3c92b2bc
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 01:45:55 UTC on e7f111948823
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13299
02:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:01.305382 2244 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:01.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:01.307473 2244 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:08:01.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:01.325862 2259 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:01.759711 2227 heartbeater.cc:499] Master 127.1.87.188:36907 was elected leader, sending a full tablet report...
02:08:02.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:01.327179 2262 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:02.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:02.643872 2263 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:08:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:02.645478 2264 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:02.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:02.648999 2244 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.324s user 0.464s sys 0.854s
02:08:02.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:02.649480 2244 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.325s user 0.468s sys 0.854s
02:08:02.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.649852 2244 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.651688 2244 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.655019 2244 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.656591 2244 hybrid_clock.cc:648] HybridClock initialized: now 1786500482656478 us; error 84 us; skew 500 ppm
02:08:02.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.658020 2244 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:02.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.668656 2244 webserver.cc:533] Webserver started at http://127.1.87.130:35735/ using document root <none> and password file <none>
02:08:02.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.670347 2244 fs_manager.cc:362] Metadata directory not provided
02:08:02.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.670701 2244 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:02.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.671653 2244 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.679423 2244 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/data/instance:
02:08:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb4e7e22e5f540e69d9c2b5f284f2e51"
02:08:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:02 on dist-test-slave-j2vl"
02:08:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.681259 2244 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/wal/instance:
02:08:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb4e7e22e5f540e69d9c2b5f284f2e51"
02:08:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:02 on dist-test-slave-j2vl"
02:08:02.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.693101 2244 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.007s sys 0.003s
02:08:02.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.702575 2282 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:02.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.704284 2244 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.004s sys 0.001s
02:08:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.704758 2244 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/data,/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/wal
02:08:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb4e7e22e5f540e69d9c2b5f284f2e51"
02:08:02.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:02 on dist-test-slave-j2vl"
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.705262 2244 fs_report.cc:389] FS layout report
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/wal
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/wal
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/data/data
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.813094 2244 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.815227 2244 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.815704 2244 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.818668 2244 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:08:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.823299 2244 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:08:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.823521 2244 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:08:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.823724 2244 ts_tablet_manager.cc:616] Registered 0 tablets
02:08:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.823863 2244 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:08:02.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.985785 2244 rpc_server.cc:307] RPC server started. Bound to: 127.1.87.130:35085
02:08:02.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.985901 2491 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.87.130:35085 every 8 connection(s)
02:08:02.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.989280 2244 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/data/info.pb
02:08:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.992769 1374 external_mini_cluster.cc:1434] Started /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu as pid 2244
02:08:02.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:02.993732 1374 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-1/wal/instance
02:08:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.017246 1374 external_mini_cluster.cc:1372] Running /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/wal
02:08:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/data
02:08:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/logs
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/data/info.pb
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.87.131:0
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.87.131
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.87.131
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.87.190:33775,127.1.87.189:38973,127.1.87.188:36907
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.87.148:42747
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:08:03.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.050040 2503 heartbeater.cc:344] Connected to a master server at 127.1.87.190:33775
02:08:03.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.050654 2503 heartbeater.cc:461] Registering TS with master...
02:08:03.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.051476 2502 heartbeater.cc:344] Connected to a master server at 127.1.87.189:38973
02:08:03.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.051812 2502 heartbeater.cc:461] Registering TS with master...
02:08:03.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.052228 2503 heartbeater.cc:507] Master 127.1.87.190:33775 requested a full tablet report, sending...
02:08:03.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.052913 2502 heartbeater.cc:507] Master 127.1.87.189:38973 requested a full tablet report, sending...
02:08:03.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.054538 2494 heartbeater.cc:344] Connected to a master server at 127.1.87.188:36907
02:08:03.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.054930 2494 heartbeater.cc:461] Registering TS with master...
02:08:03.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.055891 2494 heartbeater.cc:507] Master 127.1.87.188:36907 requested a full tablet report, sending...
02:08:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.057009 1451 ts_manager.cc:194] Registered new tserver with Master: cb4e7e22e5f540e69d9c2b5f284f2e51 (127.1.87.130:35085)
02:08:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.057989 1596 ts_manager.cc:194] Registered new tserver with Master: cb4e7e22e5f540e69d9c2b5f284f2e51 (127.1.87.130:35085)
02:08:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.058311 1762 ts_manager.cc:194] Registered new tserver with Master: cb4e7e22e5f540e69d9c2b5f284f2e51 (127.1.87.130:35085)
02:08:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.060119 1762 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.87.130:55777
02:08:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:03.433283 2519 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:03.434054 2519 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:03.434506 2519 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:03.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:03.472499 2519 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:03.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:03.473404 2519 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.87.131
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.515595 2519 tablet_server_runner.cc:78] Tablet server non-default flags:
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/logs
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.87.148:42747
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/data
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/wal
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.87.131:0
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/data/info.pb
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.87.131
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.87.190:33775,127.1.87.189:38973,127.1.87.188:36907
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.87.131
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d9153c248733781bf69132f2d094d9d3c92b2bc
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 01:45:55 UTC on e7f111948823
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13299
02:08:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.517944 2519 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:03.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:03.519853 2519 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:08:03.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:03.534593 2527 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:04.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:04.063710 2494 heartbeater.cc:499] Master 127.1.87.188:36907 was elected leader, sending a full tablet report...
02:08:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:03.535401 2529 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:04.939078 2526 debug-util.cc:398] Leaking SignalData structure 0x7b080002ede0 after lost signal to thread 2519
02:08:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:05.391304 2526 kernel_stack_watchdog.cc:198] Thread 2519 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
02:08:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:08:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:08:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:08:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:08:05.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:05.391791 2519 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.857s user 0.647s sys 1.151s
02:08:05.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:05.392325 2519 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.858s user 0.648s sys 1.151s
02:08:05.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:05.393324 2533 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:05.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:05.395587 2530 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1854 milliseconds
02:08:05.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.395710 2519 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:05.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.397678 2519 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.400554 2519 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:05.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.402010 2519 hybrid_clock.cc:648] HybridClock initialized: now 1786500485401959 us; error 69 us; skew 500 ppm
02:08:05.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.402966 2519 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:05.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.410132 2519 webserver.cc:533] Webserver started at http://127.1.87.131:40663/ using document root <none> and password file <none>
02:08:05.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.411250 2519 fs_manager.cc:362] Metadata directory not provided
02:08:05.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.411545 2519 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:05.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.412102 2519 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.418727 2519 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/data/instance:
02:08:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da082a4da315494b955356097a8187a8"
02:08:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:05 on dist-test-slave-j2vl"
02:08:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.420039 2519 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/wal/instance:
02:08:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da082a4da315494b955356097a8187a8"
02:08:05.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:05 on dist-test-slave-j2vl"
02:08:05.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.429049 2519 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
02:08:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.436764 2550 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.438126 2519 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
02:08:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.438540 2519 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/data,/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/wal
02:08:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da082a4da315494b955356097a8187a8"
02:08:05.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:05 on dist-test-slave-j2vl"
02:08:05.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.438963 2519 fs_report.cc:389] FS layout report
02:08:05.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/wal
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/wal
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/data/data
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:05.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.519708 2519 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:05.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.522277 2519 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:05.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.522971 2519 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:05.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.527758 2519 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:08:05.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.533074 2519 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:08:05.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.533304 2519 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:08:05.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.533628 2519 ts_tablet_manager.cc:616] Registered 0 tablets
02:08:05.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.533774 2519 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:08:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.692898 2519 rpc_server.cc:307] RPC server started. Bound to: 127.1.87.131:34883
02:08:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.693033 2746 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.87.131:34883 every 8 connection(s)
02:08:05.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.695842 2519 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/data/info.pb
02:08:05.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.706607 1374 external_mini_cluster.cc:1434] Started /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu as pid 2519
02:08:05.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.707227 1374 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster472343432167562872/ts-2/wal/instance
02:08:05.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.755473 2747 heartbeater.cc:344] Connected to a master server at 127.1.87.188:36907
02:08:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.756068 2747 heartbeater.cc:461] Registering TS with master...
02:08:05.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.757575 2747 heartbeater.cc:507] Master 127.1.87.188:36907 requested a full tablet report, sending...
02:08:05.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.761019 1760 ts_manager.cc:194] Registered new tserver with Master: da082a4da315494b955356097a8187a8 (127.1.87.131:34883)
02:08:05.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.763307 2748 heartbeater.cc:344] Connected to a master server at 127.1.87.189:38973
02:08:05.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.763705 2748 heartbeater.cc:461] Registering TS with master...
02:08:05.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.763643 1760 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.87.131:55129
02:08:05.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.764428 2749 heartbeater.cc:344] Connected to a master server at 127.1.87.190:33775
02:08:05.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.764796 2749 heartbeater.cc:461] Registering TS with master...
02:08:05.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.764846 2748 heartbeater.cc:507] Master 127.1.87.189:38973 requested a full tablet report, sending...
02:08:05.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.765825 2749 heartbeater.cc:507] Master 127.1.87.190:33775 requested a full tablet report, sending...
02:08:05.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.767800 1596 ts_manager.cc:194] Registered new tserver with Master: da082a4da315494b955356097a8187a8 (127.1.87.131:34883)
02:08:05.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.768412 1451 ts_manager.cc:194] Registered new tserver with Master: da082a4da315494b955356097a8187a8 (127.1.87.131:34883)
02:08:05.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:05.789454 1374 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
02:08:05.803 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:08:05.803 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:08:05.804 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.87.190"
port: 33775
}
bound_http_address {
host: "127.1.87.190"
port: 41773
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.87.189"
port: 38973
}
bound_http_address {
host: "127.1.87.189"
port: 43483
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.87.188"
port: 36907
}
bound_http_address {
host: "127.1.87.188"
port: 34493
}
}
}
02:08:05.805 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:08:05.806 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.1.87.129"
port: 41995
}
bound_http_address {
host: "127.1.87.129"
port: 42699
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.1.87.130"
port: 35085
}
bound_http_address {
host: "127.1.87.130"
port: 35735
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.1.87.131"
port: 34883
}
bound_http_address {
host: "127.1.87.131"
port: 40663
}
}
}
02:08:05.806 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:08:05.808 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster17637664005193277677/krb5kdc/krb5cc
02:08:05.808 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:08:05.810 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task0HCHY_/test-tmp
02:08:05.810 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:08:05.810 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task0HCHY_/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:08:05.814 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:08:06.183 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:06.183 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:08:06.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:06.187141 2766 mini_oidc.cc:150] Starting JWKS server
02:08:06.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:06.190480 2766 webserver.cc:533] Webserver started at http://127.0.0.1:33745/ using document root <none> and password file <none>
02:08:06.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:06.191120 2766 mini_oidc.cc:186] Starting OIDC Discovery server
02:08:06.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:06.193060 2766 webserver.cc:533] Webserver started at http://127.0.0.1:36473/ using document root <none> and password file <none>
02:08:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:08:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/principal' for realm 'KRBTEST.COM',
02:08:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:08:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:06 dist-test-slave-j2vl krb5kdc[2785](info): setting up network...
02:08:06.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:08:06.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:06 dist-test-slave-j2vl krb5kdc[2785](info): set up 2 sockets
02:08:06.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:06 dist-test-slave-j2vl krb5kdc[2785](info): commencing operation
02:08:06.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:08:06.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:06.768636 2747 heartbeater.cc:499] Master 127.1.87.188:36907 was elected leader, sending a full tablet report...
02:08:08.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:08.746686 2766 mini_kdc.cc:121] Time spent starting KDC: real 2.509s user 0.001s sys 0.026s
02:08:08.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:08:08.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:08:08.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:08:08.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:08:08.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:08:08.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:08:08.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:08:08.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:08:08.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:08:08.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:08:08.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/test-user.keytab.
02:08:08.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/test-user.keytab.
02:08:08.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:08 dist-test-slave-j2vl krb5kdc[2785](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500488, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:08.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:08:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T02:08:08Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:08:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T02:08:08Z Disabled control of system clock
02:08:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.179.190@KRBTEST.COM; defaulting to no policy
02:08:08.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:08.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.179.190@KRBTEST.COM" created.
02:08:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.179.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab.
02:08:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.179.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab.
02:08:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.179.190@KRBTEST.COM; defaulting to no policy
02:08:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.179.190@KRBTEST.COM" created.
02:08:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.179.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab.
02:08:09.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.179.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab.
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:09.048142 2766 external_mini_cluster.cc:1372] Running /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/logs
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data/info.pb
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:09.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.179.190:33289
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.179.190
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.179.190
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.179.148:43789
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.179.190:33289,127.2.179.189:34071,127.2.179.188:46699
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33745/jwks.json
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:09.442806 2814 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:09.443548 2814 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:09.444275 2814 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:09.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:09.481894 2814 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:09.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:09.482156 2814 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:09.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:09.482429 2814 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:09.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:09.482633 2814 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:09.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:09.482843 2814 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:09.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:09.483054 2814 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:09.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:09.483301 2814 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33745/jwks.json
02:08:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:09.525485 2814 master_runner.cc:387] Master server non-default flags:
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/logs
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.179.148:43789
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.179.190:33289,127.2.179.189:34071,127.2.179.188:46699
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.179.190
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.179.190:33289
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33745/jwks.json
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data/info.pb
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.179.190
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d9153c248733781bf69132f2d094d9d3c92b2bc
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 01:45:55 UTC on e7f111948823
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13299
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:09.526959 2814 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:09.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:09.528044 2814 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:08:09.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:09.540939 2821 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:09.542810 2822 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:11.473428 2814 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.932s user 0.638s sys 1.251s
02:08:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:11.473798 2814 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.932s user 0.638s sys 1.252s
02:08:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:10.946053 2820 debug-util.cc:398] Leaking SignalData structure 0x7b0800028cc0 after lost signal to thread 2814
02:08:11.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:11.478572 2823 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1936 milliseconds
02:08:11.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:11.480464 2827 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:11.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.480515 2814 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:11.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.482051 2814 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.484673 2814 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:11.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.486109 2814 hybrid_clock.cc:648] HybridClock initialized: now 1786500491486059 us; error 58 us; skew 500 ppm
02:08:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.487112 2814 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:11.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:11 dist-test-slave-j2vl krb5kdc[2785](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500491, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.494069 2814 init.cc:377] Logged in from keytab as kudu/127.2.179.190@KRBTEST.COM (short username kudu)
02:08:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.500775 2814 webserver.cc:533] Webserver started at http://127.2.179.190:33781/ using document root <none> and password file <none>
02:08:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.501920 2814 fs_manager.cc:362] Metadata directory not provided
02:08:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.502159 2814 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:11.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.502622 2814 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.507853 2814 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data/instance:
02:08:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "96c650dc9a5b447fa259ae55169f19dd"
02:08:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:11 on dist-test-slave-j2vl"
02:08:11.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.509016 2814 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal/instance:
02:08:11.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "96c650dc9a5b447fa259ae55169f19dd"
02:08:11.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:11 on dist-test-slave-j2vl"
02:08:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.518292 2814 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.001s sys 0.006s
02:08:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.526371 2848 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.527860 2814 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.004s sys 0.001s
02:08:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.528219 2814 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data,/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal
02:08:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "96c650dc9a5b447fa259ae55169f19dd"
02:08:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:11 on dist-test-slave-j2vl"
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.528554 2814 fs_report.cc:389] FS layout report
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data/data
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.582062 2814 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:11.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.593648 2814 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:11.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.594151 2814 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:11.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.693419 2814 rpc_server.cc:307] RPC server started. Bound to: 127.2.179.190:33289
02:08:11.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.693491 2936 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.179.190:33289 every 8 connection(s)
02:08:11.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.696542 2814 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data/info.pb
02:08:11.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.702073 2766 external_mini_cluster.cc:1434] Started /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu as pid 2814
02:08:11.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.702662 2766 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal/instance
02:08:11.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.705192 2939 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:08:11.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.717449 2939 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } has no permanent_uuid. Determining permanent_uuid...
02:08:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.179.189@KRBTEST.COM; defaulting to no policy
02:08:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.179.189@KRBTEST.COM" created.
02:08:11.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:11 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500491, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.190@KRBTEST.COM for kudu/127.2.179.190@KRBTEST.COM
02:08:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.179.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab.
02:08:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.179.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab.
02:08:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.765468 2939 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } has no permanent_uuid. Determining permanent_uuid...
02:08:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:11.768354 2854 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:11.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:11.773120 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.189:34071: Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:11.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.179.189@KRBTEST.COM; defaulting to no policy
02:08:11.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:11.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.179.189@KRBTEST.COM" created.
02:08:11.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.179.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab.
02:08:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.179.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab.
02:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.812803 2766 external_mini_cluster.cc:1372] Running /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal
02:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data
02:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/logs
02:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data/info.pb
02:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.179.189:34071
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.179.189
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.179.189
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.179.148:43789
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.179.190:33289,127.2.179.189:34071,127.2.179.188:46699
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33745/jwks.json
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:11.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.822834 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } attempt: 1
02:08:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:11.827821 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.189:34071: Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:11.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.896370 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } attempt: 2
02:08:11.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:11.901748 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.189:34071: Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:11.993259 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } attempt: 3
02:08:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:11.998683 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.189:34071: Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:12.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:12.142215 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } attempt: 4
02:08:12.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.149552 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.189:34071: Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:12.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.190623 2959 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:12.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.191447 2959 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.192164 2959 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:12.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.249513 2959 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:12.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.249859 2959 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:12.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.250317 2959 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:12.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.250646 2959 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:12.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.251015 2959 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:12.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.251250 2959 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:12.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.251410 2959 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33745/jwks.json
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:12.296141 2959 master_runner.cc:387] Master server non-default flags:
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/logs
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.179.148:43789
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.179.190:33289,127.2.179.189:34071,127.2.179.188:46699
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.179.189
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.179.189:34071
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33745/jwks.json
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data/info.pb
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.179.189
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d9153c248733781bf69132f2d094d9d3c92b2bc
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 01:45:55 UTC on e7f111948823
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13299
02:08:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:12.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:12.297763 2959 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:12.298992 2959 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:08:12.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.312031 2967 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:12.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:12.449164 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } attempt: 5
02:08:12.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.458724 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.189:34071: Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:13.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.006392 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } attempt: 6
02:08:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:13.021580 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.189:34071: Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:13.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:12.313437 2969 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:13.768019 2973 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:13.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:13.770529 2972 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1452 milliseconds
02:08:13.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.770639 2959 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:13.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.772148 2959 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:13.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.775785 2959 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.777570 2959 hybrid_clock.cc:648] HybridClock initialized: now 1786500493777472 us; error 103 us; skew 500 ppm
02:08:13.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.778465 2959 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:13.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:13 dist-test-slave-j2vl krb5kdc[2785](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500493, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:13.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.786123 2959 init.cc:377] Logged in from keytab as kudu/127.2.179.189@KRBTEST.COM (short username kudu)
02:08:13.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.792375 2959 webserver.cc:533] Webserver started at http://127.2.179.189:37797/ using document root <none> and password file <none>
02:08:13.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.793390 2959 fs_manager.cc:362] Metadata directory not provided
02:08:13.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.793624 2959 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:13.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.794031 2959 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.798687 2959 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data/instance:
02:08:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe6925582e504a2d86bb421d4094b359"
02:08:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:13 on dist-test-slave-j2vl"
02:08:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.799926 2959 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal/instance:
02:08:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe6925582e504a2d86bb421d4094b359"
02:08:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:13 on dist-test-slave-j2vl"
02:08:13.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.808498 2959 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.002s sys 0.008s
02:08:13.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.815369 2997 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.816820 2959 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
02:08:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.817173 2959 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data,/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal
02:08:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe6925582e504a2d86bb421d4094b359"
02:08:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:13 on dist-test-slave-j2vl"
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.817597 2959 fs_report.cc:389] FS layout report
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data/data
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:13.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:13.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.879276 2959 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:13.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.889945 2959 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:13.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.890615 2959 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.981865 2959 rpc_server.cc:307] RPC server started. Bound to: 127.2.179.189:34071
02:08:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.982088 3082 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.179.189:34071 every 8 connection(s)
02:08:13.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.986311 2959 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data/info.pb
02:08:13.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.987104 2766 external_mini_cluster.cc:1434] Started /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu as pid 2959
02:08:13.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.987651 2766 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal/instance
02:08:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:13.995990 3083 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:08:14.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.179.188@KRBTEST.COM; defaulting to no policy
02:08:14.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.010675 3083 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } has no permanent_uuid. Determining permanent_uuid...
02:08:14.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:14.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.179.188@KRBTEST.COM" created.
02:08:14.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:14 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500493, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.189@KRBTEST.COM for kudu/127.2.179.190@KRBTEST.COM
02:08:14.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.179.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab.
02:08:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.179.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab.
02:08:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.059476 3083 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } has no permanent_uuid. Determining permanent_uuid...
02:08:14.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.179.188@KRBTEST.COM; defaulting to no policy
02:08:14.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:14.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.179.188@KRBTEST.COM" created.
02:08:14.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:14 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500493, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.189@KRBTEST.COM for kudu/127.2.179.189@KRBTEST.COM
02:08:14.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.082445 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } attempt: 7
02:08:14.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.095412 3083 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } has no permanent_uuid. Determining permanent_uuid...
02:08:14.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.100040 3002 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.105777 3083 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.179.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab.
02:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.179.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab.
02:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:14 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500491, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.190@KRBTEST.COM for kudu/127.2.179.189@KRBTEST.COM
02:08:14.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.111099 2766 external_mini_cluster.cc:1372] Running /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:14.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/wal
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/logs
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data/info.pb
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.179.188:46699
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.179.188
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.179.188
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.179.148:43789
02:08:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.179.190:33289,127.2.179.189:34071,127.2.179.188:46699
02:08:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33745/jwks.json
02:08:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:14.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.123739 2939 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } has no permanent_uuid. Determining permanent_uuid...
02:08:14.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.131333 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:14.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.155457 3083 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 1
02:08:14.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.160899 3083 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:14.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.189790 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 1
02:08:14.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.195051 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.229465 3083 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 2
02:08:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.234073 3083 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.276815 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 2
02:08:14.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.287763 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:14.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.325826 3083 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 3
02:08:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.330610 3083 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.373323 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 3
02:08:14.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.378539 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:14.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.474148 3083 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 4
02:08:14.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.479099 3083 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:14.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.519052 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 4
02:08:14.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.524132 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:14.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.529933 3109 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:14.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.530537 3109 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.531309 3109 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:14.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.566566 3109 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:14.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.566826 3109 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:14.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.567390 3109 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:14.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.567873 3109 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:14.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.568209 3109 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:14.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.568449 3109 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:14.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.568602 3109 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33745/jwks.json
02:08:14.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.610420 3109 master_runner.cc:387] Master server non-default flags:
02:08:14.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/logs
02:08:14.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:14.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:14.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.179.148:43789
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/wal
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.179.190:33289,127.2.179.189:34071,127.2.179.188:46699
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.179.188
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.179.188:46699
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33745/jwks.json
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data/info.pb
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.179.188
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d9153c248733781bf69132f2d094d9d3c92b2bc
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 01:45:55 UTC on e7f111948823
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13299
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.611756 3109 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:14.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.612763 3109 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:08:14.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.625094 3122 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:14.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.778674 3083 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 5
02:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.789445 3083 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:14.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:14.807670 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 5
02:08:14.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.818501 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:15.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:15.317607 1513 debug-util.cc:398] Leaking SignalData structure 0x7b08000701a0 after lost signal to thread 1396
02:08:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:15.318706 1513 debug-util.cc:398] Leaking SignalData structure 0x7b08000a6ee0 after lost signal to thread 1520
02:08:15.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:15.337139 3083 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 6
02:08:15.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:15.369359 3083 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:15.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:15.371150 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 6
02:08:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:15.412375 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:16.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.429935 3083 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 7
02:08:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.446076 2939 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } attempt: 7
02:08:16.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:16.477720 3083 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:16.501986 2939 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.179.188:46699: Network error: Client connection negotiation failed: client connection to 127.2.179.188:46699: connect: Connection refused (error 111)
02:08:16.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:16.657675 3109 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.031s user 0.600s sys 1.183s
02:08:16.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:16.658118 3109 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.032s user 0.600s sys 1.183s
02:08:16.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:16.039451 3118 debug-util.cc:398] Leaking SignalData structure 0x7b0800028cc0 after lost signal to thread 3109
02:08:16.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:14.627187 3125 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:16.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:16.661806 3129 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.670074 3109 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:16.670053 3127 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2042 milliseconds
02:08:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.673060 3109 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:16.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.676309 3109 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:16.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.677953 3109 hybrid_clock.cc:648] HybridClock initialized: now 1786500496677888 us; error 61 us; skew 500 ppm
02:08:16.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.679545 3109 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:16 dist-test-slave-j2vl krb5kdc[2785](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500496, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:16.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.687527 3109 init.cc:377] Logged in from keytab as kudu/127.2.179.188@KRBTEST.COM (short username kudu)
02:08:16.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.707849 3109 webserver.cc:533] Webserver started at http://127.2.179.188:45157/ using document root <none> and password file <none>
02:08:16.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.709439 3109 fs_manager.cc:362] Metadata directory not provided
02:08:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.709748 3109 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.710402 3109 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:08:16.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.716708 3109 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data/instance:
02:08:16.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "31c726d560754b468f0df01a3e9e0914"
02:08:16.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:16 on dist-test-slave-j2vl"
02:08:16.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.717991 3109 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/wal/instance:
02:08:16.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "31c726d560754b468f0df01a3e9e0914"
02:08:16.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:16 on dist-test-slave-j2vl"
02:08:16.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.729480 3109 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.005s sys 0.006s
02:08:16.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.736812 3162 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.738426 3109 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.001s sys 0.005s
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.738837 3109 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data,/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/wal
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "31c726d560754b468f0df01a3e9e0914"
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:16 on dist-test-slave-j2vl"
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.739226 3109 fs_report.cc:389] FS layout report
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/wal
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/wal
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data/data
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:16.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:16.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:16.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:16.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:16.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.804656 3109 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:16.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.815932 3109 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:16.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.816660 3109 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:16.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.926486 3109 rpc_server.cc:307] RPC server started. Bound to: 127.2.179.188:46699
02:08:16.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.926674 3246 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.179.188:46699 every 8 connection(s)
02:08:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.929745 3109 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data/info.pb
02:08:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.935652 2766 external_mini_cluster.cc:1434] Started /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu as pid 3109
02:08:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.936254 2766 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/wal/instance
02:08:16.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.942140 3247 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:08:16.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:16.969467 3247 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } has no permanent_uuid. Determining permanent_uuid...
02:08:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:17 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500496, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.188@KRBTEST.COM for kudu/127.2.179.190@KRBTEST.COM
02:08:17.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.059480 3247 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } has no permanent_uuid. Determining permanent_uuid...
02:08:17.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:17 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500488, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.179.188@KRBTEST.COM
02:08:17.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:17 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500496, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.188@KRBTEST.COM for kudu/127.2.179.189@KRBTEST.COM
02:08:17.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.098696 3247 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } has no permanent_uuid. Determining permanent_uuid...
02:08:17.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:17 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500496, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.188@KRBTEST.COM for kudu/127.2.179.188@KRBTEST.COM
02:08:17.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:17 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500488, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.179.189@KRBTEST.COM
02:08:17.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.138984 3247 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: Bootstrap starting.
02:08:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.149096 3247 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: Neither blocks nor log segments found. Creating new log.
02:08:17.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.151947 3247 log.cc:826] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: Log is configured to *not* fsync() on all Append() calls
02:08:17.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:17 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500488, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.179.190@KRBTEST.COM
02:08:17.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.159256 3247 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: No bootstrap required, opened a new log
02:08:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.179519 2766 external_mini_cluster.cc:955] 0 TS(s) registered with all masters
02:08:17.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.193688 3247 raft_consensus.cc:359] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } }
02:08:17.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.194753 3247 raft_consensus.cc:385] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:08:17.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.195081 3247 raft_consensus.cc:740] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c726d560754b468f0df01a3e9e0914, State: Initialized, Role: FOLLOWER
02:08:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.196075 3247 consensus_queue.cc:260] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } }
02:08:17.197 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:17.198 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:08:17.199 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:08:17.199 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:08:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.199975 3263 sys_catalog.cc:455] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } } }
02:08:17.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.201006 3263 sys_catalog.cc:458] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [sys.catalog]: This master's current role is: FOLLOWER
02:08:17.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.202033 3247 sys_catalog.cc:565] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [sys.catalog]: configured and running, proceeding with master startup.
02:08:17.203 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.179.190"
port: 33289
}
bound_http_address {
host: "127.2.179.190"
port: 33781
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.179.189"
port: 34071
}
bound_http_address {
host: "127.2.179.189"
port: 37797
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.179.188"
port: 46699
}
bound_http_address {
host: "127.2.179.188"
port: 45157
}
}
}
02:08:17.203 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:08:17.203 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:17.203 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:08:17.203 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:08:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:17 dist-test-slave-j2vl krb5kdc[2785](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500497, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:08:17.281 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:17.283 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5cc
02:08:17.290 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:08:17.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:17.296466 3290 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:17.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:17.298869 3290 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:17.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.284816 3109 master_runner.cc:187] Error getting master registration for 127.2.179.190:33289: OK, instance_id { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" instance_seqno: 1786500491667423 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:08:17.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.322485 3109 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:08:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:17.324091 3172 tablet.cc:2406] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:08:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.332119 3263 raft_consensus.cc:493] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:17.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.333142 3263 raft_consensus.cc:515] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } }
02:08:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:17.336731 3263 leader_election.cc:290] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289), fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071)
02:08:18.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.028720 1819 debug-util.cc:398] Leaking SignalData structure 0x7b0800070400 after lost signal to thread 1682
02:08:18.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.029536 1819 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0020 after lost signal to thread 1830
02:08:18.291 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.179.190:33289
02:08:18.292 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:08:18.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:18.292620 2766 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu with pid 2814
02:08:18.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.344961 3290 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:18.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.347276 3290 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:18.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.371865 3166 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.2.179.190:33289 (error 108)
02:08:18.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.389606 3166 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289): Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111)
02:08:18.391 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:18.392 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.179.188:46699
02:08:18.392 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:08:18.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:18.392647 2766 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu with pid 3109
02:08:18.452 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:18.452 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.179.189:34071
02:08:18.452 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:08:18.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:18.452837 2766 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu with pid 2959
02:08:18.494 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:18.494 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.179.190:33289
02:08:18.494 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:08:18.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:18.496383 2766 external_mini_cluster.cc:1372] Running /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:18.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:18.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal
02:08:18.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data
02:08:18.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:18.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:18.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:18.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:18.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:18.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:18.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/logs
02:08:18.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data/info.pb
02:08:18.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:18.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:18.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.179.190:33289
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.179.190
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33781
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.179.190
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.179.148:43789
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.179.190:33289,127.2.179.189:34071,127.2.179.188:46699
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33745/jwks.json
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:18.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.867578 3298 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:18.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.868188 3298 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:18.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.869212 3298 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:18.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.910148 3298 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:18.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.910403 3298 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:18.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.910689 3298 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:18.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.910983 3298 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:18.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.911204 3298 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:18.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.911475 3298 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:18.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.911659 3298 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33745/jwks.json
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:18.956066 3298 master_runner.cc:387] Master server non-default flags:
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/logs
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.179.148:43789
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.179.190:33289,127.2.179.189:34071,127.2.179.188:46699
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.179.190
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.179.190:33289
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33745/jwks.json
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data/info.pb
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.179.190
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33781
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d9153c248733781bf69132f2d094d9d3c92b2bc
02:08:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 01:45:55 UTC on e7f111948823
02:08:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13299
02:08:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:18.957813 3298 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:18.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:18.959198 3298 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:08:18.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.973179 3315 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:20.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.019044 1819 debug-util.cc:398] Leaking SignalData structure 0x7b0800070780 after lost signal to thread 1682
02:08:20.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.020025 1819 debug-util.cc:398] Leaking SignalData structure 0x7b08000707a0 after lost signal to thread 1830
02:08:20.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.975894 3318 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:20.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.288575 3320 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:08:20.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:18.980749 3323 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:20.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.289707 3298 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:20.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.293867 3298 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:20.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.296627 3298 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:20.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.298072 3298 hybrid_clock.cc:648] HybridClock initialized: now 1786500500298043 us; error 49 us; skew 500 ppm
02:08:20.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.299019 3298 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:20.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:20 dist-test-slave-j2vl krb5kdc[2785](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500500, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:20.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.305295 3298 init.cc:377] Logged in from keytab as kudu/127.2.179.190@KRBTEST.COM (short username kudu)
02:08:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.310719 3298 webserver.cc:533] Webserver started at http://127.2.179.190:33781/ using document root <none> and password file <none>
02:08:20.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.311731 3298 fs_manager.cc:362] Metadata directory not provided
02:08:20.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.311947 3298 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:20.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.320406 3298 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
02:08:20.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.325631 3339 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:20.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.327169 3298 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
02:08:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.327539 3298 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data,/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal
02:08:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "96c650dc9a5b447fa259ae55169f19dd"
02:08:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:11 on dist-test-slave-j2vl"
02:08:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.330030 3298 fs_report.cc:389] FS layout report
02:08:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal
02:08:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal
02:08:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data/data
02:08:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:20.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:20.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:20.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:20.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:20.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:20.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:20.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:20.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.406276 3298 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:20.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.412993 3298 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:20.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.413558 3298 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:20.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.500372 3298 rpc_server.cc:307] RPC server started. Bound to: 127.2.179.190:33289
02:08:20.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.500512 3421 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.179.190:33289 every 8 connection(s)
02:08:20.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.503695 3298 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/data/info.pb
02:08:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.511075 2766 external_mini_cluster.cc:1434] Started /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu as pid 3298
02:08:20.511 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:20.511 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.179.188:46699
02:08:20.512 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.513499 2766 external_mini_cluster.cc:1372] Running /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/wal
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/logs
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data/info.pb
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.179.188:46699
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.179.188
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45157
02:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.179.188
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.179.148:43789
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.179.190:33289,127.2.179.189:34071,127.2.179.188:46699
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33745/jwks.json
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.519556 3427 sys_catalog.cc:263] Verifying existing consensus state
02:08:20.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260812 02:08:20.520885 3427 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:08:20.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.521539 3298 master.cc:562] Master@127.2.179.190:33289 shutting down...
02:08:20.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.623667 3298 master.cc:584] Master@127.2.179.190:33289 shutdown complete.
02:08:20.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.626881 3298 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
02:08:20.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:08:20.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.897413 3433 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.898115 3433 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.898804 3433 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:20.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.937877 3433 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:20.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.938146 3433 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:20.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.938441 3433 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:20.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.938648 3433 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:20.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.938858 3433 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:20.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.939160 3433 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:20.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:20.939483 3433 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33745/jwks.json
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.984792 3433 master_runner.cc:387] Master server non-default flags:
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/logs
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.179.148:43789
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/wal
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.179.190:33289,127.2.179.189:34071,127.2.179.188:46699
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.179.188
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.179.188:46699
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33745/jwks.json
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data/info.pb
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.179.188
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45157
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d9153c248733781bf69132f2d094d9d3c92b2bc
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 01:45:55 UTC on e7f111948823
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13299
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.986675 3433 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:20.987917 3433 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:08:21.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:21.004509 3451 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:22.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:22.407981 3446 debug-util.cc:398] Leaking SignalData structure 0x7b0800028cc0 after lost signal to thread 3433
02:08:22.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:22.696189 3456 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1691 milliseconds
02:08:22.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:21.004628 3448 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:22.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:22.695530 3433 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.692s user 0.550s sys 0.990s
02:08:22.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:22.697172 3433 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.694s user 0.550s sys 0.990s
02:08:22.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:22.698472 3458 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.698912 3433 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:22.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.702965 3433 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:22.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.706949 3433 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:22.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.708462 3433 hybrid_clock.cc:648] HybridClock initialized: now 1786500502708428 us; error 48 us; skew 500 ppm
02:08:22.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.709601 3433 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:22.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:22 dist-test-slave-j2vl krb5kdc[2785](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500502, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:22.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.717741 3433 init.cc:377] Logged in from keytab as kudu/127.2.179.188@KRBTEST.COM (short username kudu)
02:08:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.722766 3433 webserver.cc:533] Webserver started at http://127.2.179.188:45157/ using document root <none> and password file <none>
02:08:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.723697 3433 fs_manager.cc:362] Metadata directory not provided
02:08:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.723923 3433 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:22.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.732620 3433 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.005s
02:08:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.738333 3483 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:22.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.739885 3433 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
02:08:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.740255 3433 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data,/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/wal
02:08:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "31c726d560754b468f0df01a3e9e0914"
02:08:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:16 on dist-test-slave-j2vl"
02:08:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.742375 3433 fs_report.cc:389] FS layout report
02:08:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/wal
02:08:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/wal
02:08:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data/data
02:08:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:22.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.795038 3433 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:22.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.805721 3433 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:22.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.806188 3433 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.887873 3433 rpc_server.cc:307] RPC server started. Bound to: 127.2.179.188:46699
02:08:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.888013 3581 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.179.188:46699 every 8 connection(s)
02:08:22.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.891763 3433 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-2/data/info.pb
02:08:22.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.901721 2766 external_mini_cluster.cc:1434] Started /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu as pid 3433
02:08:22.902 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:22.902 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.179.189:34071
02:08:22.902 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.903712 3582 sys_catalog.cc:263] Verifying existing consensus state
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.904305 2766 external_mini_cluster.cc:1372] Running /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/logs
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data/info.pb
02:08:22.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.179.189:34071
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.179.189
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37797
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.179.189
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.179.148:43789
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.179.190:33289,127.2.179.189:34071,127.2.179.188:46699
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33745/jwks.json
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:22.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.909884 3582 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: Bootstrap starting.
02:08:22.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.964008 3582 log.cc:826] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: Log is configured to *not* fsync() on all Append() calls
02:08:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.970070 3582 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:08:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:22.971035 3582 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: Bootstrap complete.
02:08:23.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.001523 3582 raft_consensus.cc:359] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } }
02:08:23.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.003875 3582 raft_consensus.cc:385] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:08:23.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.004197 3582 raft_consensus.cc:740] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c726d560754b468f0df01a3e9e0914, State: Initialized, Role: FOLLOWER
02:08:23.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.005244 3582 consensus_queue.cc:260] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } }
02:08:23.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.008649 3588 sys_catalog.cc:455] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } } }
02:08:23.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.009614 3588 sys_catalog.cc:458] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [sys.catalog]: This master's current role is: FOLLOWER
02:08:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.010444 3582 sys_catalog.cc:565] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [sys.catalog]: configured and running, proceeding with master startup.
02:08:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.049431 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:23.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.050181 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.062265 3498 tablet.cc:2406] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:08:23.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.054862 3433 master_runner.cc:187] Error getting master registration for 127.2.179.190:33289: Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111),
02:08:23.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.085642 3433 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:08:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.186399 3588 raft_consensus.cc:493] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.187002 3588 raft_consensus.cc:515] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } }
02:08:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.190035 3588 leader_election.cc:290] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289), fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071)
02:08:23.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.194857 3489 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111)
02:08:23.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.201597 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289): Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111)
02:08:23.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.202701 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071): Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:23.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.203271 3489 leader_election.cc:304] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 31c726d560754b468f0df01a3e9e0914; no voters: 96c650dc9a5b447fa259ae55169f19dd, fe6925582e504a2d86bb421d4094b359
02:08:23.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.204634 3588 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:08:23.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.377084 3585 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:08:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.377827 3585 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.378710 3585 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:08:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.421099 3585 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.421470 3585 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:08:23.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.421799 3585 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:08:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.422009 3585 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.422223 3585 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:08:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.422624 3585 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.422863 3585 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:33745/jwks.json
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.509806 3585 master_runner.cc:387] Master server non-default flags:
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/logs
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.179.148:43789
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.179.190:33289,127.2.179.189:34071,127.2.179.188:46699
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/kudu.keytab
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.179.189
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.179.189:34071
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33745/jwks.json
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data/info.pb
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.179.189
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37797
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 1d9153c248733781bf69132f2d094d9d3c92b2bc
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Aug 2026 01:45:55 UTC on e7f111948823
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 13299
02:08:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:23.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.512012 3585 env_posix.cc:2285] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:23.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:23.514004 3585 file_cache.cc:504] Constructed file cache file cache with capacity 419430
02:08:23.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.529143 3636 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:24.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:24.055234 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:24.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:24.055620 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:24.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:24.782356 3646 raft_consensus.cc:493] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:24.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:24.783150 3646 raft_consensus.cc:515] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } }
02:08:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:24.802031 3646 leader_election.cc:290] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289), fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071)
02:08:24.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:24.807241 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289): Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111)
02:08:24.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:24.812158 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071): Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:24.813325 3489 leader_election.cc:304] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 31c726d560754b468f0df01a3e9e0914; no voters: 96c650dc9a5b447fa259ae55169f19dd, fe6925582e504a2d86bb421d4094b359
02:08:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:24.814762 3646 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:08:25.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:25.060225 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:25.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:25.060636 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:24.932137 3635 debug-util.cc:398] Leaking SignalData structure 0x7b0800028cc0 after lost signal to thread 3585
02:08:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:23.532832 3637 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:25.621189 3585 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.091s user 0.718s sys 1.095s
02:08:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:25.623584 3585 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.093s user 0.718s sys 1.095s
02:08:25.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:25.623672 3638 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2090 milliseconds
02:08:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:25.624673 3639 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.624603 3585 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.629772 3585 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:25.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.633442 3585 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.634966 3585 hybrid_clock.cc:648] HybridClock initialized: now 1786500505634826 us; error 154 us; skew 500 ppm
02:08:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.636132 3585 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:25 dist-test-slave-j2vl krb5kdc[2785](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500505, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.642817 3585 init.cc:377] Logged in from keytab as kudu/127.2.179.189@KRBTEST.COM (short username kudu)
02:08:25.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.655745 3585 webserver.cc:533] Webserver started at http://127.2.179.189:37797/ using document root <none> and password file <none>
02:08:25.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.657037 3585 fs_manager.cc:362] Metadata directory not provided
02:08:25.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.657323 3585 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:25.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.669153 3585 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.005s sys 0.001s
02:08:25.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.674881 3666 log_block_manager.cc:4127] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:25.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.676484 3585 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
02:08:25.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.676932 3585 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data,/tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal
02:08:25.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fe6925582e504a2d86bb421d4094b359"
02:08:25.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-12 02:08:13 on dist-test-slave-j2vl"
02:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.679276 3585 fs_report.cc:389] FS layout report
02:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal
02:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal
02:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data/data
02:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:25.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.779021 3585 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:25.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.789114 3585 env_posix.cc:2285] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:25.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.789796 3585 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:25.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.888463 3585 rpc_server.cc:307] RPC server started. Bound to: 127.2.179.189:34071
02:08:25.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.888643 3746 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.179.189:34071 every 8 connection(s)
02:08:25.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.892635 3585 server_base.cc:1193] Dumped server information to /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/data/info.pb
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.896739 2766 external_mini_cluster.cc:1434] Started /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu as pid 3585
02:08:25.897 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:25.904 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:25.906 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.179.188:46699(127.2.179.188:46699)] connecting to peer
02:08:25.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.910905 3748 sys_catalog.cc:263] Verifying existing consensus state
02:08:25.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260812 02:08:25.912249 3748 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:08:25.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:25.912869 3585 master.cc:562] Master@127.2.179.189:34071 shutting down...
02:08:25.913 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:25.914 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.2.179.188:46699(127.2.179.188:46699)] Successfully connected to peer
02:08:25.919 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:25.919 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:25.919 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:25.924 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
02:08:25.924 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5cc
02:08:25.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:25 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1786500497, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:25.929 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.2.179.189:34071(127.2.179.189:34071)] Successfully connected to peer
02:08:25.933 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:08:25.933 [INFO - kudu-nio-1] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
02:08:25.935 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.2.179.188
02:08:25.949 [DEBUG - kudu-nio-2] (SecurityContext.java:222) Refreshing Kerberos credentials...
02:08:25.950 [DEBUG - kudu-nio-2] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/krb5kdc/krb5cc
02:08:25.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:25 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1786500497, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:25.959 [DEBUG - kudu-nio-2] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:08:25.959 [INFO - kudu-nio-2] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
02:08:25.960 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.2.179.189
02:08:25.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:25 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1786500497, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.179.188@KRBTEST.COM
02:08:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:26 dist-test-slave-j2vl krb5kdc[2785](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1786500497, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.179.189@KRBTEST.COM
02:08:26.045 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0x34313f1a, L:/127.0.0.1:56618 - R:/127.2.179.188:46699] using SASL/GSSAPI
02:08:26.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:26.065833 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:26.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:26.066190 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:26.080 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0x59ef2506, L:/127.0.0.1:42916 - R:/127.2.179.189:34071] using SASL/GSSAPI
02:08:26.091 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Service unavailable: service kudu.master.MasterService not registered on Master
02:08:26.091 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:26.093 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.2.179.189:34071(127.2.179.189:34071)] handling channelInactive
02:08:26.093 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state READY due to: connection closed
02:08:26.105 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:26.112 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:26.113 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.114 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.114 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.117 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.117 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.118 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.118 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:26.125 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:26.126 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:26.132 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.132 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.133 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.138 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.138 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.138 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.138 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:26.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:26.143299 3585 master.cc:584] Master@127.2.179.189:34071 shutdown complete.
02:08:26.144 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:26.145 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:26.146 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.146 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.146 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.146 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:26.148566 3585 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
02:08:26.150 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:26.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0HCHY_/test-tmp/mini-kudu-cluster4623634467175290696/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:08:26.164 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:26.165 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:26.166 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.166 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.167 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.168 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.169 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.169 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.169 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:26.204 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:26.205 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:26.206 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.206 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.206 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.207 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.207 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.207 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.211 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:26.244 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:26.245 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:26.246 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.246 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.246 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.246 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.247 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.247 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.249 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:26.304 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:26.305 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:26.305 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.305 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.305 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.306 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.306 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.311 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:26.384 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:26.386 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.386 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:26.388 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.388 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.388 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.390 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:26.545 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:26.546 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:26.547 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.547 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.547 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:26.547 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.547 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.547 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:26.549 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:27.070036 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:27.070334 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:27.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:27.159703 3778 raft_consensus.cc:493] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:27.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:27.160175 3778 raft_consensus.cc:515] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } }
02:08:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:27.162009 3778 leader_election.cc:290] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289), fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071)
02:08:27.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:27.174842 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289): Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111)
02:08:27.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:27.175493 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071): Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:27.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:27.175869 3489 leader_election.cc:304] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 31c726d560754b468f0df01a3e9e0914; no voters: 96c650dc9a5b447fa259ae55169f19dd, fe6925582e504a2d86bb421d4094b359
02:08:27.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:27.176474 3778 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:08:27.585 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:27.586 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:27.586 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:27.586 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:27.586 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:27.587 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:27.587 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:27.587 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:27.589 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:28.074299 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:28.074625 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:28.088958 3433 master_runner.cc:187] Error getting master registration for 127.2.179.190:33289: Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111),
02:08:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:28.089453 3433 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:08:29.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:29.078241 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:29.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:29.078595 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:29.244 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:29.246 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:29.246 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:29.246 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:29.246 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:29.247 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:29.248 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:29.248 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:29.249 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:29.864 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:29.865 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:29.866 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:29.866 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:29.866 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:29.867 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:29.867 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:29.867 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:29.869 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:30.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:30.082413 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:30.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:30.082736 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:30.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:30.926312 3790 raft_consensus.cc:493] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:30.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:30.926985 3790 raft_consensus.cc:515] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } }
02:08:30.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:30.928848 3790 leader_election.cc:290] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289), fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071)
02:08:30.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:30.932725 3489 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111) [suppressed 5 similar messages]
02:08:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:30.937767 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289): Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111)
02:08:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:30.938539 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071): Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:30.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:30.939023 3489 leader_election.cc:304] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 31c726d560754b468f0df01a3e9e0914; no voters: 96c650dc9a5b447fa259ae55169f19dd, fe6925582e504a2d86bb421d4094b359
02:08:30.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:30.939813 3790 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:08:31.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:31.086702 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:31.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:31.087073 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:32.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:32.090925 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:32.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:32.091517 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:32.104 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:32.106 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:32.106 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:32.106 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:32.107 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:32.107 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:32.107 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:32.107 [INFO - pool-44-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:32.109 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:32.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:32 dist-test-slave-j2vl krb5kdc[2785](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500512, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:32.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:32.722173 3473 init.cc:295] Successfully reacquired a new kerberos TGT
02:08:33.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:33.092656 3433 master_runner.cc:187] Error getting master registration for 127.2.179.190:33289: Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111),
02:08:33.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:33.093333 3433 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:08:33.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:33.095008 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:33.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:33.095248 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:34.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:34.098765 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:34.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:34.099061 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:35.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:35.102155 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:35.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:35.102456 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:35.845 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:35.846 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:35.846 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:35.846 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:35.847 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:35.847 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:35.847 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:35.847 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:35.850 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:36.105259 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:36.105618 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:37.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:37.108873 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:37.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:37.109269 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:37.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:37.652514 3807 raft_consensus.cc:493] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:37.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:37.652925 3807 raft_consensus.cc:515] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } }
02:08:37.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:37.654904 3807 leader_election.cc:290] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289), fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071)
02:08:37.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:37.658387 3489 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111) [suppressed 1 similar messages]
02:08:37.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:37.661834 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289): Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111)
02:08:37.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:37.662532 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071): Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:37.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:37.662943 3489 leader_election.cc:304] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 31c726d560754b468f0df01a3e9e0914; no voters: 96c650dc9a5b447fa259ae55169f19dd, fe6925582e504a2d86bb421d4094b359
02:08:37.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:37.663658 3807 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:08:38.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:38.095896 3433 master_runner.cc:187] Error getting master registration for 127.2.179.190:33289: Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111),
02:08:38.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:38.096315 3433 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:08:38.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:38.112505 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:38.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:38.112797 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:38.224 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:38.225 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:38.226 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:38.226 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:38.226 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:38.226 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:38.227 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:38.227 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:38.229 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:39.116153 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:39.116453 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:39.565 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:39.566 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:39.567 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:39.567 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:39.567 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:39.567 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:39.567 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:39.567 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:39.569 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:40.119704 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:40.120117 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:41.005 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:41.006 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:41.006 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:41.006 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:41.006 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:41.006 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:41.007 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:41.007 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:41.010 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:41.124104 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:41.124645 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:41.344 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:41.345 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:41.345 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:41.346 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:41.346 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:41.346 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:41.347 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:41.347 [INFO - pool-44-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:41.349 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:42.128767 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:42.129165 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:42 dist-test-slave-j2vl krb5kdc[2785](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500522, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:42.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:42.725086 3473 init.cc:295] Successfully reacquired a new kerberos TGT
02:08:43.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:43.100984 3433 master_runner.cc:187] Error getting master registration for 127.2.179.190:33289: Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111),
02:08:43.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:43.101680 3433 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:08:43.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:43.132572 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:43.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:43.132905 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:44.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:44.136313 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:44.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:44.136680 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:44.545 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:44.546 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:44.546 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:44.546 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:44.546 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:44.547 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:44.547 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:44.547 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:44.550 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:45.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:45.140406 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:45.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:45.140692 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:45.924 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:45.925 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:45.926 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:45.926 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:45.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:45.927 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:45.927 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:45.927 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:45.929 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:46.143734 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:46.144030 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:47.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:47.147231 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:47.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:47.147810 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:47.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:47.508869 3831 raft_consensus.cc:493] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:47.509501 3831 raft_consensus.cc:515] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } }
02:08:47.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:47.511497 3831 leader_election.cc:290] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289), fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071)
02:08:47.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:47.515113 3489 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111) [suppressed 1 similar messages]
02:08:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:47.519186 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289): Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111)
02:08:47.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:47.519781 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071): Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:47.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:47.520061 3489 leader_election.cc:304] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 31c726d560754b468f0df01a3e9e0914; no voters: 96c650dc9a5b447fa259ae55169f19dd, fe6925582e504a2d86bb421d4094b359
02:08:47.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:47.520882 3831 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:08:48.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:48.104964 3433 master_runner.cc:187] Error getting master registration for 127.2.179.190:33289: Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111),
02:08:48.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:48.105404 3433 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:08:48.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:48.151664 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:48.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:48.151964 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:48.484 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:48.485 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:48.486 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:48.486 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:48.486 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:48.486 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:48.486 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:48.487 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:48.488 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:48.704 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:48.706 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:48.706 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:48.706 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:48.706 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:48.707 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:48.707 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:48.707 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:48.708 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:49.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:49.155069 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:49.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:49.155352 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:49.964094 3847 raft_consensus.cc:493] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:49.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:49.964634 3847 raft_consensus.cc:515] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "96c650dc9a5b447fa259ae55169f19dd" member_type: VOTER last_known_addr { host: "127.2.179.190" port: 33289 } } peers { permanent_uuid: "fe6925582e504a2d86bb421d4094b359" member_type: VOTER last_known_addr { host: "127.2.179.189" port: 34071 } } peers { permanent_uuid: "31c726d560754b468f0df01a3e9e0914" member_type: VOTER last_known_addr { host: "127.2.179.188" port: 46699 } }
02:08:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:49.966379 3847 leader_election.cc:290] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289), fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071)
02:08:49.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:49.972295 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 96c650dc9a5b447fa259ae55169f19dd (127.2.179.190:33289): Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111)
02:08:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:49.972918 3489 leader_election.cc:336] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer fe6925582e504a2d86bb421d4094b359 (127.2.179.189:34071): Network error: Client connection negotiation failed: client connection to 127.2.179.189:34071: connect: Connection refused (error 111)
02:08:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:49.973248 3489 leader_election.cc:304] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 31c726d560754b468f0df01a3e9e0914; no voters: 96c650dc9a5b447fa259ae55169f19dd, fe6925582e504a2d86bb421d4094b359
02:08:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:49.973963 3847 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:08:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:50.158911 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:50.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:50.159225 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:50.764 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:50.765 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:50.765 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:50.765 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:50.766 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:50.766 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:50.766 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:50.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:50.768 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:51.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:51.163494 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:51.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:51.163959 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:51.205 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:51.206 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:51.206 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:51.207 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:51.207 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:51.207 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:51.207 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:51.207 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:51.210 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:52.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:52.167367 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:52.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:52.167716 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:52.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 12 02:08:52 dist-test-slave-j2vl krb5kdc[2785](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786500532, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.179.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:52.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:52.728060 3473 init.cc:295] Successfully reacquired a new kerberos TGT
02:08:53.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:53.109306 3433 master_runner.cc:187] Error getting master registration for 127.2.179.190:33289: Network error: Client connection negotiation failed: client connection to 127.2.179.190:33289: connect: Connection refused (error 111),
02:08:53.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:53.109830 3433 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:08:53.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:53.170598 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:53.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:53.170925 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:54.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:54.173604 3607 catalog_manager.cc:1594] T 00000000000000000000000000000000 P 31c726d560754b468f0df01a3e9e0914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:54.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260812 02:08:54.173894 3607 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:55.004 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.190:33289(127.2.179.190:33289)] connecting to peer
02:08:55.005 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.2.179.189:34071(127.2.179.189:34071)] connecting to peer
02:08:55.005 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:55.006 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.2.179.190:33289(127.2.179.190:33289)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:55.006 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:55.006 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.2.179.189:34071(127.2.179.189:34071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:55.006 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.190:33289: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289
02:08:55.006 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.2.179.189:34071: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
02:08:55.008 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071; will retry
02:08:55.010 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29111), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [16ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [148ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [188ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [188ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [201ms] refreshing cache from master, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [208ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [209ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [211ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [212ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [214ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [215ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [226ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [229ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [235ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [236ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [243ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [243ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [246ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [247ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [261ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [263ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [266ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [266ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [266ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [304ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [304ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [308ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [309ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [341ms] refreshing cache from master, [341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [343ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [344ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [346ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [346ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [400ms] refreshing cache from master, [401ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [402ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [403ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [408ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [408ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [481ms] refreshing cache from master, [481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [483ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [485ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [487ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [487ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [641ms] refreshing cache from master, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [644ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [644ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [646ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [647ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [1681ms] refreshing cache from master, [1682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [1682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [1683ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [1686ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [1686ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3341ms] refreshing cache from master, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3346ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [3346ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3961ms] refreshing cache from master, [3961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [3962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [3962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [3964ms] too many traces: truncated at 100 traces, [3964ms] too many traces: truncated at 100 traces, deferred=Deferred@815920402(state=PENDING, result=null, callback=(continuation of Deferred@370685163 after retry RPC after error@1299818137) -> (continuation of Deferred@415794428 after retry RPC after error@671017874) -> (continuation of Deferred@1888894186 after retry RPC after error@1893400847) -> (continuation of Deferred@745051276 after retry RPC after error@2082101480) -> (continuation of Deferred@1026181869 after retry RPC after error@366077758) -> (continuation of Deferred@2113265148 after retry RPC after error@1401292722) -> (continuation of Deferred@1016353293 after retry RPC after error@652651190) -> (continuation of Deferred@1974442411 after retry RPC after error@376941371) -> (continuation of Deferred@149784514 after retry RPC after error@466431859) -> (continuation of Deferred@304521056 after retry RPC after error@958644332) -> (continuation of Deferred@1570013981 after retry RPC after error@802195819) -> (continuation of Deferred@153869934 after retry RPC after error@774655879) -> (continuation of Deferred@1465498932 after retry RPC after error@2095709541) -> (continuation of Deferred@736527416 after retry RPC after error@1736095412) -> (continuation of Deferred@2102400882 after retry RPC after error@1315037032) -> (continuation of Deferred@1362707928 after retry RPC after error@1665439325) -> (continuation of Deferred@517138112 after retry RPC after error@1586032877) -> (continuation of Deferred@1157917025 after retry RPC after error@37091810) -> (continuation of Deferred@198769538 after retry RPC after error@315518348) -> (continuation of Deferred@1952924117 after retry RPC after error@1496337490) -> (continuation of Deferred@368617501 after retry RPC after error@1950029091) -> (continuation of Deferred@983337880 after retry RPC after error@1628766789) -> (continuation of Deferred@838723130 after retry RPC after error@490958635) -> (continuation of Deferred@1130231632 after retry RPC after error@169071188) -> (continuation of Deferred@1417750810 after retry RPC after error@1379741334), errback=(continuation of Deferred@370685163 after retry RPC after error@1299818137) -> (continuation of Deferred@415794428 after retry RPC after error@671017874) -> (continuation of Deferred@1888894186 after retry RPC after error@1893400847) -> (continuation of Deferred@745051276 after retry RPC after error@2082101480) -> (continuation of Deferred@1026181869 after retry RPC after error@366077758) -> (continuation of Deferred@2113265148 after retry RPC after error@1401292722) -> (continuation of Deferred@1016353293 after retry RPC after error@652651190) -> (continuation of Deferred@1974442411 after retry RPC after error@376941371) -> (continuation of Deferred@149784514 after retry RPC after error@466431859) -> (continuation of Deferred@304521056 after retry RPC after error@958644332) -> (continuation of Deferred@1570013981 after retry RPC after error@802195819) -> (continuation of Deferred@153869934 after retry RPC after error@774655879) -> (continuation of Deferred@1465498932 after retry RPC after error@2095709541) -> (continuation of Deferred@736527416 after retry RPC after error@1736095412) -> (continuation of Deferred@2102400882 after retry RPC after error@1315037032) -> (continuation of Deferred@1362707928 after retry RPC after error@1665439325) -> (continuation of Deferred@517138112 after retry RPC after error@1586032877) -> (continuation of Deferred@1157917025 after retry RPC after error@37091810) -> (continuation of Deferred@198769538 after retry RPC after error@315518348) -> (continuation of Deferred@1952924117 after retry RPC after error@1496337490) -> (continuation of Deferred@368617501 after retry RPC after error@1950029091) -> (continuation of Deferred@983337880 after retry RPC after error@1628766789) -> (continuation of Deferred@838723130 after retry RPC after error@490958635) -> (continuation of Deferred@1130231632 after retry RPC after error@169071188) -> (continuation of Deferred@1417750810 after retry RPC after error@1379741334)))
02:08:55.012 [DEBUG - main] (Connection.java:575) [peer master-127.2.179.188:46699(127.2.179.188:46699)] disconnecting while in state READY
02:08:55.012 [DEBUG - main] (Connection.java:575) [peer master-127.2.179.190:33289(127.2.179.190:33289)] disconnecting while in state TERMINATED
02:08:55.012 [DEBUG - main] (Connection.java:575) [peer master-127.2.179.189:34071(127.2.179.189:34071)] disconnecting while in state TERMINATED
02:08:55.013 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:08:55.014 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.2.179.188:46699(127.2.179.188:46699)] handling channelInactive
02:08:55.014 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.179.188:46699(127.2.179.188:46699)] cleaning up while in state READY due to: connection closed
02:08:55.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:55.015882 2766 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu with pid 3433
02:08:56.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T02:08:56Z chronyd exiting
02:08:57.332 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:08:57.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:57.335160 1374 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu with pid 1923
02:08:57.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:57.413870 1374 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu with pid 2244
02:08:57.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:57.459710 1374 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu with pid 2519
02:08:57.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:57.555111 1374 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu with pid 1395
02:08:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:57.607340 1374 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu with pid 1527
02:08:57.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260812 02:08:57.663966 1374 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task0HCHY_/build/tsan/bin/kudu with pid 1675
02:08:58.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-12T02:08:58Z chronyd exiting
02:08:59.942 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29111), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [16ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [148ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [188ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [188ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [201ms] refreshing cache from master, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [208ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [209ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [211ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [212ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [214ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [215ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [226ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [229ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [235ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [236ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [243ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [243ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [246ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [247ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [261ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [263ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [266ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [266ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [266ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [304ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [304ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [308ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [309ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [341ms] refreshing cache from master, [341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [343ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [344ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [346ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [346ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [400ms] refreshing cache from master, [401ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [402ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [403ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [408ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [408ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [481ms] refreshing cache from master, [481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [483ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [485ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [487ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [487ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [641ms] refreshing cache from master, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [644ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [644ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [646ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [647ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [1681ms] refreshing cache from master, [1682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [1682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [1683ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [1686ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [1686ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3341ms] refreshing cache from master, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3346ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [3346ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3961ms] refreshing cache from master, [3961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [3962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [3962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [3964ms] too many traces: truncated at 100 traces, [3964ms] too many traces: truncated at 100 traces, deferred=Deferred@815920402(state=PENDING, result=null, callback=(continuation of Deferred@370685163 after retry RPC after error@1299818137) -> (continuation of Deferred@415794428 after retry RPC after error@671017874) -> (continuation of Deferred@1888894186 after retry RPC after error@1893400847) -> (continuation of Deferred@745051276 after retry RPC after error@2082101480) -> (continuation of Deferred@1026181869 after retry RPC after error@366077758) -> (continuation of Deferred@2113265148 after retry RPC after error@1401292722) -> (continuation of Deferred@1016353293 after retry RPC after error@652651190) -> (continuation of Deferred@1974442411 after retry RPC after error@376941371) -> (continuation of Deferred@149784514 after retry RPC after error@466431859) -> (continuation of Deferred@304521056 after retry RPC after error@958644332) -> (continuation of Deferred@1570013981 after retry RPC after error@802195819) -> (continuation of Deferred@153869934 after retry RPC after error@774655879) -> (continuation of Deferred@1465498932 after retry RPC after error@2095709541) -> (continuation of Deferred@736527416 after retry RPC after error@1736095412) -> (continuation of Deferred@2102400882 after retry RPC after error@1315037032) -> (continuation of Deferred@1362707928 after retry RPC after error@1665439325) -> (continuation of Deferred@517138112 after retry RPC after error@1586032877) -> (continuation of Deferred@1157917025 after retry RPC after error@37091810) -> (continuation of Deferred@198769538 after retry RPC after error@315518348) -> (continuation of Deferred@1952924117 after retry RPC after error@1496337490) -> (continuation of Deferred@368617501 after retry RPC after error@1950029091) -> (continuation of Deferred@983337880 after retry RPC after error@1628766789) -> (continuation of Deferred@838723130 after retry RPC after error@490958635) -> (continuation of Deferred@1130231632 after retry RPC after error@169071188) -> (continuation of Deferred@1417750810 after retry RPC after error@1379741334), errback=(continuation of Deferred@370685163 after retry RPC after error@1299818137) -> (continuation of Deferred@415794428 after retry RPC after error@671017874) -> (continuation of Deferred@1888894186 after retry RPC after error@1893400847) -> (continuation of Deferred@745051276 after retry RPC after error@2082101480) -> (continuation of Deferred@1026181869 after retry RPC after error@366077758) -> (continuation of Deferred@2113265148 after retry RPC after error@1401292722) -> (continuation of Deferred@1016353293 after retry RPC after error@652651190) -> (continuation of Deferred@1974442411 after retry RPC after error@376941371) -> (continuation of Deferred@149784514 after retry RPC after error@466431859) -> (continuation of Deferred@304521056 after retry RPC after error@958644332) -> (continuation of Deferred@1570013981 after retry RPC after error@802195819) -> (continuation of Deferred@153869934 after retry RPC after error@774655879) -> (continuation of Deferred@1465498932 after retry RPC after error@2095709541) -> (continuation of Deferred@736527416 after retry RPC after error@1736095412) -> (continuation of Deferred@2102400882 after retry RPC after error@1315037032) -> (continuation of Deferred@1362707928 after retry RPC after error@1665439325) -> (continuation of Deferred@517138112 after retry RPC after error@1586032877) -> (continuation of Deferred@1157917025 after retry RPC after error@37091810) -> (continuation of Deferred@198769538 after retry RPC after error@315518348) -> (continuation of Deferred@1952924117 after retry RPC after error@1496337490) -> (continuation of Deferred@368617501 after retry RPC after error@1950029091) -> (continuation of Deferred@983337880 after retry RPC after error@1628766789) -> (continuation of Deferred@838723130 after retry RPC after error@490958635) -> (continuation of Deferred@1130231632 after retry RPC after error@169071188) -> (continuation of Deferred@1417750810 after retry RPC after error@1379741334)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:348)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.2.179.189:34071
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29111), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [16ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [148ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [188ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [188ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [201ms] refreshing cache from master, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [208ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [209ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [211ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [212ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [214ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [215ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [226ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [229ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [235ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [236ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [243ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [243ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [246ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [247ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [261ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [263ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [266ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [266ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [266ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [304ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [304ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [308ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [309ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [341ms] refreshing cache from master, [341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [343ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [344ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [346ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [346ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [400ms] refreshing cache from master, [401ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [402ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [403ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [408ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [408ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [481ms] refreshing cache from master, [481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [483ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [485ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [487ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [487ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [641ms] refreshing cache from master, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [644ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [644ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [646ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [647ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [1681ms] refreshing cache from master, [1682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [1682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [1683ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [1686ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [1686ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3341ms] refreshing cache from master, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3346ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [3346ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3961ms] refreshing cache from master, [3961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [3962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [3962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [3964ms] too many traces: truncated at 100 traces, [3964ms] too many traces: truncated at 100 traces, deferred=Deferred@815920402(state=PENDING, result=null, callback=(continuation of Deferred@370685163 after retry RPC after error@1299818137) -> (continuation of Deferred@415794428 after retry RPC after error@671017874) -> (continuation of Deferred@1888894186 after retry RPC after error@1893400847) -> (continuation of Deferred@745051276 after retry RPC after error@2082101480) -> (continuation of Deferred@1026181869 after retry RPC after error@366077758) -> (continuation of Deferred@2113265148 after retry RPC after error@1401292722) -> (continuation of Deferred@1016353293 after retry RPC after error@652651190) -> (continuation of Deferred@1974442411 after retry RPC after error@376941371) -> (continuation of Deferred@149784514 after retry RPC after error@466431859) -> (continuation of Deferred@304521056 after retry RPC after error@958644332) -> (continuation of Deferred@1570013981 after retry RPC after error@802195819) -> (continuation of Deferred@153869934 after retry RPC after error@774655879) -> (continuation of Deferred@1465498932 after retry RPC after error@2095709541) -> (continuation of Deferred@736527416 after retry RPC after error@1736095412) -> (continuation of Deferred@2102400882 after retry RPC after error@1315037032) -> (continuation of Deferred@1362707928 after retry RPC after error@1665439325) -> (continuation of Deferred@517138112 after retry RPC after error@1586032877) -> (continuation of Deferred@1157917025 after retry RPC after error@37091810) -> (continuation of Deferred@198769538 after retry RPC after error@315518348) -> (continuation of Deferred@1952924117 after retry RPC after error@1496337490) -> (continuation of Deferred@368617501 after retry RPC after error@1950029091) -> (continuation of Deferred@983337880 after retry RPC after error@1628766789) -> (continuation of Deferred@838723130 after retry RPC after error@490958635) -> (continuation of Deferred@1130231632 after retry RPC after error@169071188) -> (continuation of Deferred@1417750810 after retry RPC after error@1379741334), errback=(continuation of Deferred@370685163 after retry RPC after error@1299818137) -> (continuation of Deferred@415794428 after retry RPC after error@671017874) -> (continuation of Deferred@1888894186 after retry RPC after error@1893400847) -> (continuation of Deferred@745051276 after retry RPC after error@2082101480) -> (continuation of Deferred@1026181869 after retry RPC after error@366077758) -> (continuation of Deferred@2113265148 after retry RPC after error@1401292722) -> (continuation of Deferred@1016353293 after retry RPC after error@652651190) -> (continuation of Deferred@1974442411 after retry RPC after error@376941371) -> (continuation of Deferred@149784514 after retry RPC after error@466431859) -> (continuation of Deferred@304521056 after retry RPC after error@958644332) -> (continuation of Deferred@1570013981 after retry RPC after error@802195819) -> (continuation of Deferred@153869934 after retry RPC after error@774655879) -> (continuation of Deferred@1465498932 after retry RPC after error@2095709541) -> (continuation of Deferred@736527416 after retry RPC after error@1736095412) -> (continuation of Deferred@2102400882 after retry RPC after error@1315037032) -> (continuation of Deferred@1362707928 after retry RPC after error@1665439325) -> (continuation of Deferred@517138112 after retry RPC after error@1586032877) -> (continuation of Deferred@1157917025 after retry RPC after error@37091810) -> (continuation of Deferred@198769538 after retry RPC after error@315518348) -> (continuation of Deferred@1952924117 after retry RPC after error@1496337490) -> (continuation of Deferred@368617501 after retry RPC after error@1950029091) -> (continuation of Deferred@983337880 after retry RPC after error@1628766789) -> (continuation of Deferred@838723130 after retry RPC after error@490958635) -> (continuation of Deferred@1130231632 after retry RPC after error@169071188) -> (continuation of Deferred@1417750810 after retry RPC after error@1379741334)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:348)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.2.179.189:34071
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
02:09:00.066 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: giving up after 1 attempts
02:09:00.067 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29111), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [16ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [148ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [188ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [188ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [201ms] refreshing cache from master, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [208ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [209ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [211ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [212ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [214ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [215ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [226ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [229ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [235ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [236ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [243ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [243ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [246ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [247ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [261ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [263ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [266ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [266ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [266ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [304ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [304ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [308ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [309ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [341ms] refreshing cache from master, [341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [343ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [344ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [346ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [346ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [400ms] refreshing cache from master, [401ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [402ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [403ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [408ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [408ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [481ms] refreshing cache from master, [481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [483ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [485ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [487ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [487ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [641ms] refreshing cache from master, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [644ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [644ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [646ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [647ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [1681ms] refreshing cache from master, [1682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [1682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [1683ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [1686ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [1686ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3341ms] refreshing cache from master, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3346ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [3346ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3961ms] refreshing cache from master, [3961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [3962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [3962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [3964ms] too many traces: truncated at 100 traces, [3964ms] too many traces: truncated at 100 traces, deferred=Deferred@815920402(state=PENDING, result=null, callback=(continuation of Deferred@370685163 after retry RPC after error@1299818137) -> (continuation of Deferred@415794428 after retry RPC after error@671017874) -> (continuation of Deferred@1888894186 after retry RPC after error@1893400847) -> (continuation of Deferred@745051276 after retry RPC after error@2082101480) -> (continuation of Deferred@1026181869 after retry RPC after error@366077758) -> (continuation of Deferred@2113265148 after retry RPC after error@1401292722) -> (continuation of Deferred@1016353293 after retry RPC after error@652651190) -> (continuation of Deferred@1974442411 after retry RPC after error@376941371) -> (continuation of Deferred@149784514 after retry RPC after error@466431859) -> (continuation of Deferred@304521056 after retry RPC after error@958644332) -> (continuation of Deferred@1570013981 after retry RPC after error@802195819) -> (continuation of Deferred@153869934 after retry RPC after error@774655879) -> (continuation of Deferred@1465498932 after retry RPC after error@2095709541) -> (continuation of Deferred@736527416 after retry RPC after error@1736095412) -> (continuation of Deferred@2102400882 after retry RPC after error@1315037032) -> (continuation of Deferred@1362707928 after retry RPC after error@1665439325) -> (continuation of Deferred@517138112 after retry RPC after error@1586032877) -> (continuation of Deferred@1157917025 after retry RPC after error@37091810) -> (continuation of Deferred@198769538 after retry RPC after error@315518348) -> (continuation of Deferred@1952924117 after retry RPC after error@1496337490) -> (continuation of Deferred@368617501 after retry RPC after error@1950029091) -> (continuation of Deferred@983337880 after retry RPC after error@1628766789) -> (continuation of Deferred@838723130 after retry RPC after error@490958635) -> (continuation of Deferred@1130231632 after retry RPC after error@169071188) -> (continuation of Deferred@1417750810 after retry RPC after error@1379741334), errback=(continuation of Deferred@370685163 after retry RPC after error@1299818137) -> (continuation of Deferred@415794428 after retry RPC after error@671017874) -> (continuation of Deferred@1888894186 after retry RPC after error@1893400847) -> (continuation of Deferred@745051276 after retry RPC after error@2082101480) -> (continuation of Deferred@1026181869 after retry RPC after error@366077758) -> (continuation of Deferred@2113265148 after retry RPC after error@1401292722) -> (continuation of Deferred@1016353293 after retry RPC after error@652651190) -> (continuation of Deferred@1974442411 after retry RPC after error@376941371) -> (continuation of Deferred@149784514 after retry RPC after error@466431859) -> (continuation of Deferred@304521056 after retry RPC after error@958644332) -> (continuation of Deferred@1570013981 after retry RPC after error@802195819) -> (continuation of Deferred@153869934 after retry RPC after error@774655879) -> (continuation of Deferred@1465498932 after retry RPC after error@2095709541) -> (continuation of Deferred@736527416 after retry RPC after error@1736095412) -> (continuation of Deferred@2102400882 after retry RPC after error@1315037032) -> (continuation of Deferred@1362707928 after retry RPC after error@1665439325) -> (continuation of Deferred@517138112 after retry RPC after error@1586032877) -> (continuation of Deferred@1157917025 after retry RPC after error@37091810) -> (continuation of Deferred@198769538 after retry RPC after error@315518348) -> (continuation of Deferred@1952924117 after retry RPC after error@1496337490) -> (continuation of Deferred@368617501 after retry RPC after error@1950029091) -> (continuation of Deferred@983337880 after retry RPC after error@1628766789) -> (continuation of Deferred@838723130 after retry RPC after error@490958635) -> (continuation of Deferred@1130231632 after retry RPC after error@169071188) -> (continuation of Deferred@1417750810 after retry RPC after error@1379741334)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:348)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.2.179.189:34071
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29111), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [16ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [148ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [188ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [188ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [201ms] refreshing cache from master, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [208ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [209ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [211ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [212ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [214ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [215ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [226ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [229ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [235ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [236ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [243ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [243ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [246ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [247ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [261ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [263ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [266ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [266ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [266ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [304ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [304ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [308ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [309ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [341ms] refreshing cache from master, [341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [343ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [344ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [346ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [346ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [400ms] refreshing cache from master, [401ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [402ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [403ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [408ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [408ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [481ms] refreshing cache from master, [481ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [483ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [485ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [487ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [487ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [641ms] refreshing cache from master, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [644ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [644ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [646ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [647ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [1681ms] refreshing cache from master, [1682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [1682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [1683ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [1683ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [1684ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [1686ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [1686ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3341ms] refreshing cache from master, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.190:33289: Network error: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.189:34071: Network error: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3346ms] Sub RPC ConnectToMaster: received response from server master-127.2.179.188:46699: OK, [3346ms] delaying RPC due to: Service unavailable: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071, [3961ms] refreshing cache from master, [3961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.190:33289, [3962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.188:46699, [3962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.179.189:34071, [3964ms] too many traces: truncated at 100 traces, [3964ms] too many traces: truncated at 100 traces, deferred=Deferred@815920402(state=PENDING, result=null, callback=(continuation of Deferred@370685163 after retry RPC after error@1299818137) -> (continuation of Deferred@415794428 after retry RPC after error@671017874) -> (continuation of Deferred@1888894186 after retry RPC after error@1893400847) -> (continuation of Deferred@745051276 after retry RPC after error@2082101480) -> (continuation of Deferred@1026181869 after retry RPC after error@366077758) -> (continuation of Deferred@2113265148 after retry RPC after error@1401292722) -> (continuation of Deferred@1016353293 after retry RPC after error@652651190) -> (continuation of Deferred@1974442411 after retry RPC after error@376941371) -> (continuation of Deferred@149784514 after retry RPC after error@466431859) -> (continuation of Deferred@304521056 after retry RPC after error@958644332) -> (continuation of Deferred@1570013981 after retry RPC after error@802195819) -> (continuation of Deferred@153869934 after retry RPC after error@774655879) -> (continuation of Deferred@1465498932 after retry RPC after error@2095709541) -> (continuation of Deferred@736527416 after retry RPC after error@1736095412) -> (continuation of Deferred@2102400882 after retry RPC after error@1315037032) -> (continuation of Deferred@1362707928 after retry RPC after error@1665439325) -> (continuation of Deferred@517138112 after retry RPC after error@1586032877) -> (continuation of Deferred@1157917025 after retry RPC after error@37091810) -> (continuation of Deferred@198769538 after retry RPC after error@315518348) -> (continuation of Deferred@1952924117 after retry RPC after error@1496337490) -> (continuation of Deferred@368617501 after retry RPC after error@1950029091) -> (continuation of Deferred@983337880 after retry RPC after error@1628766789) -> (continuation of Deferred@838723130 after retry RPC after error@490958635) -> (continuation of Deferred@1130231632 after retry RPC after error@169071188) -> (continuation of Deferred@1417750810 after retry RPC after error@1379741334), errback=(continuation of Deferred@370685163 after retry RPC after error@1299818137) -> (continuation of Deferred@415794428 after retry RPC after error@671017874) -> (continuation of Deferred@1888894186 after retry RPC after error@1893400847) -> (continuation of Deferred@745051276 after retry RPC after error@2082101480) -> (continuation of Deferred@1026181869 after retry RPC after error@366077758) -> (continuation of Deferred@2113265148 after retry RPC after error@1401292722) -> (continuation of Deferred@1016353293 after retry RPC after error@652651190) -> (continuation of Deferred@1974442411 after retry RPC after error@376941371) -> (continuation of Deferred@149784514 after retry RPC after error@466431859) -> (continuation of Deferred@304521056 after retry RPC after error@958644332) -> (continuation of Deferred@1570013981 after retry RPC after error@802195819) -> (continuation of Deferred@153869934 after retry RPC after error@774655879) -> (continuation of Deferred@1465498932 after retry RPC after error@2095709541) -> (continuation of Deferred@736527416 after retry RPC after error@1736095412) -> (continuation of Deferred@2102400882 after retry RPC after error@1315037032) -> (continuation of Deferred@1362707928 after retry RPC after error@1665439325) -> (continuation of Deferred@517138112 after retry RPC after error@1586032877) -> (continuation of Deferred@1157917025 after retry RPC after error@37091810) -> (continuation of Deferred@198769538 after retry RPC after error@315518348) -> (continuation of Deferred@1952924117 after retry RPC after error@1496337490) -> (continuation of Deferred@368617501 after retry RPC after error@1950029091) -> (continuation of Deferred@983337880 after retry RPC after error@1628766789) -> (continuation of Deferred@838723130 after retry RPC after error@490958635) -> (continuation of Deferred@1130231632 after retry RPC after error@169071188) -> (continuation of Deferred@1417750810 after retry RPC after error@1379741334)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:348)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.2.179.190:33289,127.2.179.188:46699,127.2.179.189:34071) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.190:33289(127.2.179.190:33289): Connection refused: /127.2.179.190:33289,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:338)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.2.179.189:34071(127.2.179.189:34071): Connection refused: /127.2.179.189:34071
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.2.179.189:34071
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)