08:07:09.014 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:07:09.014 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskfpJR7l/test-tmp
08:07:09.015 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:07:09.015 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskfpJR7l/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:07:09.018 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:07:09.370 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:07:09.370 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:07:09.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:09.375135 7290 mini_oidc.cc:150] Starting JWKS server
08:07:09.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:09.378355 7290 webserver.cc:533] Webserver started at http://127.0.0.1:44269/ using document root <none> and password file <none>
08:07:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:09.378806 7290 mini_oidc.cc:186] Starting OIDC Discovery server
08:07:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:09.380538 7290 webserver.cc:533] Webserver started at http://127.0.0.1:45801/ using document root <none> and password file <none>
08:07:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T08:07:09Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:07:09.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T08:07:09Z Disabled control of system clock
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:09.445768 7290 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/wal
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/data
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/logs
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/data/info.pb
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.30.190:37805
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.30.190
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.30.148:41025
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.30.190:37805,127.7.30.189:37049,127.7.30.188:43913
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:09.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44269/jwks.json with env {}
08:07:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:09.788549 7302 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:09.789135 7302 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:09.789594 7302 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:09.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:09.823194 7302 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:07:09.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:09.823493 7302 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:09.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:09.823694 7302 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:09.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:09.823912 7302 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:07:09.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:09.824110 7302 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:09.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:09.824275 7302 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:44269/jwks.json
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:09.862036 7302 master_runner.cc:387] Master server non-default flags:
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.30.148:41025
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/data
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/wal
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.30.190:37805,127.7.30.189:37049,127.7.30.188:43913
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.30.190:37805
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44269/jwks.json
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/data/info.pb
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.30.190
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/logs
08:07:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:07:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Jun 2026 07:43:21 UTC on e7f111948823
08:07:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12467
08:07:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:09.863365 7302 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:09.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:09.864959 7302 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:09.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:09.876588 7308 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:09.876632 7309 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.134892 7311 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.140354 7310 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1262 milliseconds
08:07:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.141623 7302 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.265s user 0.367s sys 0.886s
08:07:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.142011 7302 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.266s user 0.371s sys 0.886s
08:07:11.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.142217 7302 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:11.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.143383 7302 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.146476 7302 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:11.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.147953 7302 hybrid_clock.cc:648] HybridClock initialized: now 1781856431147901 us; error 63 us; skew 500 ppm
08:07:11.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.148914 7302 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:11.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.157317 7302 webserver.cc:533] Webserver started at http://127.7.30.190:46715/ using document root <none> and password file <none>
08:07:11.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.158422 7302 fs_manager.cc:362] Metadata directory not provided
08:07:11.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.158677 7302 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:11.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.159202 7302 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:11.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.163758 7302 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/data/instance:
08:07:11.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21239e92f47d4e93945ea8a144115d03"
08:07:11.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:11 on dist-test-slave-jt9t"
08:07:11.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.165166 7302 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/wal/instance:
08:07:11.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21239e92f47d4e93945ea8a144115d03"
08:07:11.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:11 on dist-test-slave-jt9t"
08:07:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.173645 7302 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.001s
08:07:11.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.180375 7318 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:11.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.181730 7302 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
08:07:11.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.182269 7302 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/data,/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/wal
08:07:11.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21239e92f47d4e93945ea8a144115d03"
08:07:11.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:11 on dist-test-slave-jt9t"
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.182695 7302 fs_report.cc:389] FS layout report
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/wal
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/wal
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/data/data
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:11.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:11.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.279027 7302 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.280774 7302 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.281338 7302 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.360662 7302 rpc_server.cc:307] RPC server started. Bound to: 127.7.30.190:37805
08:07:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.360759 7369 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.30.190:37805 every 8 connection(s)
08:07:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.363665 7302 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/data/info.pb
08:07:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.370337 7370 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:11.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.373636 7290 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu as pid 7302
08:07:11.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.374184 7290 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-0/wal/instance
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.379624 7290 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/wal
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/data
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/logs
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/data/info.pb
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.30.189:37049
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.30.189
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.30.148:41025
08:07:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.30.190:37805,127.7.30.189:37049,127.7.30.188:43913
08:07:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44269/jwks.json with env {}
08:07:11.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.386835 7370 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } has no permanent_uuid. Determining permanent_uuid...
08:07:11.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.413987 7370 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } has no permanent_uuid. Determining permanent_uuid...
08:07:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.416741 7321 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.30.189:37049: connect: Connection refused (error 111)
08:07:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.421379 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.189:37049: Network error: Client connection negotiation failed: client connection to 127.7.30.189:37049: connect: Connection refused (error 111)
08:07:11.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.470969 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } attempt: 1
08:07:11.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.475623 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.189:37049: Network error: Client connection negotiation failed: client connection to 127.7.30.189:37049: connect: Connection refused (error 111)
08:07:11.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.544138 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } attempt: 2
08:07:11.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.548657 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.189:37049: Network error: Client connection negotiation failed: client connection to 127.7.30.189:37049: connect: Connection refused (error 111)
08:07:11.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.640180 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } attempt: 3
08:07:11.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.645470 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.189:37049: Network error: Client connection negotiation failed: client connection to 127.7.30.189:37049: connect: Connection refused (error 111)
08:07:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.709769 7372 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.710480 7372 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:11.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.711048 7372 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:11.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.744957 7372 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:07:11.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.745371 7372 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:11.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.745752 7372 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:11.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.746099 7372 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:07:11.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.746404 7372 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:11.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.746646 7372 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:44269/jwks.json
08:07:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.785176 7372 master_runner.cc:387] Master server non-default flags:
08:07:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.30.148:41025
08:07:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/data
08:07:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/wal
08:07:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.30.190:37805,127.7.30.189:37049,127.7.30.188:43913
08:07:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.30.189:37049
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44269/jwks.json
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/data/info.pb
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.30.189
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/logs
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:07:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Jun 2026 07:43:21 UTC on e7f111948823
08:07:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12467
08:07:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:11.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.786949 7372 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.788664 7372 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:11.788921 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } attempt: 4
08:07:11.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.793756 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.189:37049: Network error: Client connection negotiation failed: client connection to 127.7.30.189:37049: connect: Connection refused (error 111)
08:07:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.801415 7381 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:12.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:12.093300 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } attempt: 5
08:07:12.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:12.102329 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.189:37049: Network error: Client connection negotiation failed: client connection to 127.7.30.189:37049: connect: Connection refused (error 111)
08:07:12.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:12.649935 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } attempt: 6
08:07:12.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:12.679912 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.189:37049: Network error: Client connection negotiation failed: client connection to 127.7.30.189:37049: connect: Connection refused (error 111)
08:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.205158 7380 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 7372
08:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.262313 7380 kernel_stack_watchdog.cc:198] Thread 7372 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
08:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:07:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:07:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:11.802678 7382 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.263118 7372 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.461s user 0.551s sys 0.906s
08:07:13.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.263671 7372 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.462s user 0.551s sys 0.907s
08:07:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.265611 7384 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:13.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.268090 7383 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1459 milliseconds
08:07:13.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.268208 7372 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:13.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.270521 7372 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:13.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.273343 7372 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:13.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.274813 7372 hybrid_clock.cc:648] HybridClock initialized: now 1781856433274798 us; error 94 us; skew 500 ppm
08:07:13.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.275804 7372 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:13.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.282447 7372 webserver.cc:533] Webserver started at http://127.7.30.189:36423/ using document root <none> and password file <none>
08:07:13.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.283456 7372 fs_manager.cc:362] Metadata directory not provided
08:07:13.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.283715 7372 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:13.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.284225 7372 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:13.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.288762 7372 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/data/instance:
08:07:13.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23f40fe1eb024882a90852518d52efa9"
08:07:13.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:13 on dist-test-slave-jt9t"
08:07:13.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.290189 7372 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/wal/instance:
08:07:13.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23f40fe1eb024882a90852518d52efa9"
08:07:13.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:13 on dist-test-slave-jt9t"
08:07:13.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.298471 7372 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.002s
08:07:13.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.304952 7393 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:13.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.306301 7372 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
08:07:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.306809 7372 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/data,/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/wal
08:07:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23f40fe1eb024882a90852518d52efa9"
08:07:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:13 on dist-test-slave-jt9t"
08:07:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.307216 7372 fs_report.cc:389] FS layout report
08:07:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/wal
08:07:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/wal
08:07:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/data/data
08:07:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:13.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.362449 7372 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.364533 7372 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:13.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.365131 7372 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:13.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.439821 7372 rpc_server.cc:307] RPC server started. Bound to: 127.7.30.189:37049
08:07:13.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.439906 7444 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.30.189:37049 every 8 connection(s)
08:07:13.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.442500 7372 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/data/info.pb
08:07:13.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.445312 7290 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu as pid 7372
08:07:13.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.446136 7290 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-1/wal/instance
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.448185 7290 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/wal
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/data
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/logs
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/data/info.pb
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:13.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.30.188:43913
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.30.188
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.30.148:41025
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.30.190:37805,127.7.30.189:37049,127.7.30.188:43913
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44269/jwks.json with env {}
08:07:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.448915 7445 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.465279 7445 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } has no permanent_uuid. Determining permanent_uuid...
08:07:13.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.489988 7445 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } has no permanent_uuid. Determining permanent_uuid...
08:07:13.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.504729 7445 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } has no permanent_uuid. Determining permanent_uuid...
08:07:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.507743 7395 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:13.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.512584 7445 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:13.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.562218 7445 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 1
08:07:13.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.567569 7445 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:13.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.636054 7445 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 2
08:07:13.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.641479 7445 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:13.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.733129 7445 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 3
08:07:13.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.737958 7445 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:13.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.740561 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } attempt: 7
08:07:13.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.754261 7370 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } has no permanent_uuid. Determining permanent_uuid...
08:07:13.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.761257 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:13.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.791123 7447 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:13.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.791689 7447 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:13.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.792152 7447 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:13.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.819725 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 1
08:07:13.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.842060 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:13.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.846326 7447 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:07:13.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.846602 7447 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:13.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.846814 7447 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:13.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.847018 7447 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:07:13.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.847218 7447 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:13.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.847365 7447 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:44269/jwks.json
08:07:13.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.881371 7445 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 4
08:07:13.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.885550 7445 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.887178 7447 master_runner.cc:387] Master server non-default flags:
08:07:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.30.148:41025
08:07:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/data
08:07:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/wal
08:07:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.30.190:37805,127.7.30.189:37049,127.7.30.188:43913
08:07:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.30.188:43913
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44269/jwks.json
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/data/info.pb
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.30.188
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/logs
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Jun 2026 07:43:21 UTC on e7f111948823
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12467
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.888415 7447 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:13.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.890147 7447 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.901077 7459 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:13.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:13.923559 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 2
08:07:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.931303 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:14.016965 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 3
08:07:14.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:14.024811 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:14.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:14.165486 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 4
08:07:14.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:14.173080 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:14.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:14.185025 7445 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 5
08:07:14.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:14.192307 7445 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:14.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:14.456717 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 5
08:07:14.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:14.466333 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:14.739915 7445 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 6
08:07:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:14.755455 7445 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:14.835374 7366 debug-util.cc:398] Leaking SignalData structure 0x7b080007cbe0 after lost signal to thread 7303
08:07:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:14.836632 7366 debug-util.cc:398] Leaking SignalData structure 0x7b08000705c0 after lost signal to thread 7369
08:07:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.018915 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 6
08:07:15.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:15.053400 7370 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.30.188:43913: Network error: Client connection negotiation failed: client connection to 127.7.30.188:43913: connect: Connection refused (error 111)
08:07:15.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:13.903246 7460 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:15.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:15.363211 7462 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:15.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:15.366343 7461 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1459 milliseconds
08:07:15.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.366425 7447 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:15.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.367719 7447 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:15.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.370808 7447 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:15.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.372416 7447 hybrid_clock.cc:648] HybridClock initialized: now 1781856435372378 us; error 65 us; skew 500 ppm
08:07:15.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.373342 7447 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:15.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.380878 7447 webserver.cc:533] Webserver started at http://127.7.30.188:40917/ using document root <none> and password file <none>
08:07:15.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.381924 7447 fs_manager.cc:362] Metadata directory not provided
08:07:15.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.382176 7447 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:15.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.382676 7447 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:15.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.387281 7447 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/data/instance:
08:07:15.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a0d59d6ef0e471d85d9cc874502dde1"
08:07:15.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:15 on dist-test-slave-jt9t"
08:07:15.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.388504 7447 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/wal/instance:
08:07:15.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a0d59d6ef0e471d85d9cc874502dde1"
08:07:15.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:15 on dist-test-slave-jt9t"
08:07:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.396665 7447 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.010s sys 0.000s
08:07:15.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.403312 7473 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:15.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.404633 7447 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
08:07:15.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.405028 7447 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/data,/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/wal
08:07:15.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a0d59d6ef0e471d85d9cc874502dde1"
08:07:15.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:15 on dist-test-slave-jt9t"
08:07:15.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.405395 7447 fs_report.cc:389] FS layout report
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/wal
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/wal
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/data/data
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:15.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:15.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.476425 7447 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:15.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.478089 7447 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.478605 7447 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:15.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.567849 7447 rpc_server.cc:307] RPC server started. Bound to: 127.7.30.188:43913
08:07:15.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.567917 7524 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.30.188:43913 every 8 connection(s)
08:07:15.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.570710 7447 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/data/info.pb
08:07:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.575420 7290 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu as pid 7447
08:07:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.575848 7290 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/master-2/wal/instance
08:07:15.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.578240 7525 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:15.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.599638 7525 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } has no permanent_uuid. Determining permanent_uuid...
08:07:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.641825 7525 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } has no permanent_uuid. Determining permanent_uuid...
08:07:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.653700 7525 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } has no permanent_uuid. Determining permanent_uuid...
08:07:15.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.673413 7525 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1: Bootstrap starting.
08:07:15.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.680675 7525 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1: Neither blocks nor log segments found. Creating new log.
08:07:15.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.682642 7525 log.cc:826] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1: Log is configured to *not* fsync() on all Append() calls
08:07:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.689167 7525 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1: No bootstrap required, opened a new log
08:07:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.720834 7525 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } }
08:07:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.721892 7525 raft_consensus.cc:385] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.722265 7525 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7a0d59d6ef0e471d85d9cc874502dde1, State: Initialized, Role: FOLLOWER
08:07:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.723309 7525 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } }
08:07:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.726560 7534 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } } }
08:07:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.727259 7534 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [sys.catalog]: This master's current role is: FOLLOWER
08:07:15.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.728688 7525 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [sys.catalog]: configured and running, proceeding with master startup.
08:07:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:15.735380 7479 tablet.cc:2406] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:07:15.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:15.766388 7545 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:15.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:15.767004 7545 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:15.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.758152 7447 master_runner.cc:187] Error getting master registration for 127.7.30.190:37805: OK, instance_id { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" instance_seqno: 1781856431340846 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:07:15.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.788909 7447 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:07:15.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.816069 7445 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 7
08:07:15.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.842763 7445 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9: Bootstrap starting.
08:07:15.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.858203 7445 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9: Neither blocks nor log segments found. Creating new log.
08:07:15.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.860934 7445 log.cc:826] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9: Log is configured to *not* fsync() on all Append() calls
08:07:15.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.866817 7445 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9: No bootstrap required, opened a new log
08:07:15.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.900918 7445 raft_consensus.cc:359] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } }
08:07:15.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.901938 7445 raft_consensus.cc:385] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:15.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.902293 7445 raft_consensus.cc:740] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23f40fe1eb024882a90852518d52efa9, State: Initialized, Role: FOLLOWER
08:07:15.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.903378 7445 consensus_queue.cc:260] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } }
08:07:15.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.906673 7549 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } } }
08:07:15.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.907823 7549 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [sys.catalog]: This master's current role is: FOLLOWER
08:07:15.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.908617 7445 sys_catalog.cc:565] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [sys.catalog]: configured and running, proceeding with master startup.
08:07:15.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:15.970551 7560 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:15.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:15.971266 7560 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:15.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.944195 7372 master_runner.cc:187] Error getting master registration for 127.7.30.190:37805: OK, instance_id { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" instance_seqno: 1781856431340846 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:07:15.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:15.996079 7372 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:07:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.070767 7534 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:16.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.071369 7534 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } }
08:07:16.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.077454 7420 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "23f40fe1eb024882a90852518d52efa9" is_pre_election: true
08:07:16.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.078430 7420 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7a0d59d6ef0e471d85d9cc874502dde1 in term 0.
08:07:16.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.081341 7476 leader_election.cc:304] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23f40fe1eb024882a90852518d52efa9, 7a0d59d6ef0e471d85d9cc874502dde1; no voters:
08:07:16.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.083487 7534 leader_election.cc:290] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 21239e92f47d4e93945ea8a144115d03 (127.7.30.190:37805), 23f40fe1eb024882a90852518d52efa9 (127.7.30.189:37049)
08:07:16.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.086297 7534 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:07:16.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.086846 7370 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } attempt: 7
08:07:16.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.087086 7534 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:07:16.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.087672 7534 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [term 0 FOLLOWER]: Advancing to term 1
08:07:16.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:16.124897 7399 tablet.cc:2406] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:07:16.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.129796 7534 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } }
08:07:16.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.152916 7420 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "23f40fe1eb024882a90852518d52efa9"
08:07:16.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.153604 7420 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [term 0 FOLLOWER]: Advancing to term 1
08:07:16.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.161361 7420 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7a0d59d6ef0e471d85d9cc874502dde1 in term 1.
08:07:16.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.162787 7476 leader_election.cc:304] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23f40fe1eb024882a90852518d52efa9, 7a0d59d6ef0e471d85d9cc874502dde1; no voters:
08:07:16.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.163933 7562 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [term 1 FOLLOWER]: Leader election won for term 1
08:07:16.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.166685 7534 leader_election.cc:290] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [CANDIDATE]: Term 1 election: Requested vote from peers 21239e92f47d4e93945ea8a144115d03 (127.7.30.190:37805), 23f40fe1eb024882a90852518d52efa9 (127.7.30.189:37049)
08:07:16.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.184257 7562 raft_consensus.cc:697] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [term 1 LEADER]: Becoming Leader. State: Replica: 7a0d59d6ef0e471d85d9cc874502dde1, State: Running, Role: LEADER
08:07:16.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.185583 7562 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } }
08:07:16.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.193277 7370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03: Bootstrap starting.
08:07:16.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.201524 7534 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7a0d59d6ef0e471d85d9cc874502dde1. Latest consensus state: current_term: 1 leader_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } } }
08:07:16.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.203465 7534 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [sys.catalog]: This master's current role is: LEADER
08:07:16.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.211272 7566 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
08:07:16.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.214727 7370 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03: Neither blocks nor log segments found. Creating new log.
08:07:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.219996 7566 catalog_manager.cc:1519] Initializing Kudu cluster ID...
08:07:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.220445 7370 log.cc:826] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03: Log is configured to *not* fsync() on all Append() calls
08:07:16.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.239470 7370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03: No bootstrap required, opened a new log
08:07:16.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.263584 7420 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [term 1 FOLLOWER]: Refusing update from remote peer 7a0d59d6ef0e471d85d9cc874502dde1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:07:16.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.266150 7534 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:07:16.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.315297 7549 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7a0d59d6ef0e471d85d9cc874502dde1. Latest consensus state: current_term: 1 leader_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } } }
08:07:16.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.316129 7549 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [sys.catalog]: This master's current role is: FOLLOWER
08:07:16.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.323328 7370 raft_consensus.cc:359] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } }
08:07:16.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.325520 7370 raft_consensus.cc:385] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.325615 7568 mvcc.cc:204] Tried to move back new op lower bound from 7298483962914115584 to 7298483962628399104. Current Snapshot: MvccSnapshot[applied={T|T < 7298483962914115584}]
08:07:16.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.326058 7370 raft_consensus.cc:740] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 21239e92f47d4e93945ea8a144115d03, State: Initialized, Role: FOLLOWER
08:07:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.327142 7370 consensus_queue.cc:260] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } }
08:07:16.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.342327 7564 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } } }
08:07:16.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.343230 7564 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [sys.catalog]: This master's current role is: LEADER
08:07:16.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.344635 7569 mvcc.cc:204] Tried to move back new op lower bound from 7298483962914115584 to 7298483962628399104. Current Snapshot: MvccSnapshot[applied={T|T < 7298483962914115584}]
08:07:16.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.357376 7549 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } } }
08:07:16.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.358389 7549 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9 [sys.catalog]: This master's current role is: FOLLOWER
08:07:16.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.363828 7566 catalog_manager.cc:1382] Generated new cluster ID: dda38a83a38b482bb2076f61a3fa9bd2
08:07:16.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.364181 7566 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
08:07:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.365451 7572 sys_catalog.cc:455] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } } }
08:07:16.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.366549 7572 sys_catalog.cc:458] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03 [sys.catalog]: This master's current role is: FOLLOWER
08:07:16.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.369592 7370 sys_catalog.cc:565] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03 [sys.catalog]: configured and running, proceeding with master startup.
08:07:16.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.372218 7343 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03 [term 0 FOLLOWER]: Advancing to term 1
08:07:16.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.371150 7345 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "21239e92f47d4e93945ea8a144115d03" is_pre_election: true
08:07:16.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.370141 7344 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "21239e92f47d4e93945ea8a144115d03"
08:07:16.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.385408 7343 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03 [term 1 FOLLOWER]: Refusing update from remote peer 7a0d59d6ef0e471d85d9cc874502dde1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:07:16.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.400005 7564 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:07:16.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.463961 7566 catalog_manager.cc:1405] Generated new certificate authority record
08:07:16.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.466281 7566 catalog_manager.cc:1539] Loading token signing keys...
08:07:16.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.485589 7572 sys_catalog.cc:455] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7a0d59d6ef0e471d85d9cc874502dde1. Latest consensus state: current_term: 1 leader_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } } }
08:07:16.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.486588 7572 sys_catalog.cc:458] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03 [sys.catalog]: This master's current role is: FOLLOWER
08:07:16.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.488111 7577 sys_catalog.cc:455] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } } }
08:07:16.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.491314 7534 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "21239e92f47d4e93945ea8a144115d03" member_type: VOTER last_known_addr { host: "127.7.30.190" port: 37805 } } peers { permanent_uuid: "23f40fe1eb024882a90852518d52efa9" member_type: VOTER last_known_addr { host: "127.7.30.189" port: 37049 } } peers { permanent_uuid: "7a0d59d6ef0e471d85d9cc874502dde1" member_type: VOTER last_known_addr { host: "127.7.30.188" port: 43913 } } }
08:07:16.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.492026 7534 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1 [sys.catalog]: This master's current role is: LEADER
08:07:16.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.499882 7577 sys_catalog.cc:458] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03 [sys.catalog]: This master's current role is: FOLLOWER
08:07:16.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.582805 7566 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 7a0d59d6ef0e471d85d9cc874502dde1: Generated new TSK 0
08:07:16.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.584262 7566 catalog_manager.cc:1549] Initializing in-progress tserver states...
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.607584 7290 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/wal
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/data
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/logs
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/data/info.pb
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:07:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.30.129:0
08:07:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.30.129
08:07:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.30.129
08:07:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.30.190:37805,127.7.30.189:37049,127.7.30.188:43913
08:07:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.30.148:41025
08:07:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:07:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.683027 7589 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:07:16.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.977264 7560 catalog_manager.cc:1294] Loaded cluster ID: dda38a83a38b482bb2076f61a3fa9bd2
08:07:16.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.977871 7560 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9: loading cluster ID for follower catalog manager: success
08:07:16.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:16.991717 7560 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9: acquiring CA information for follower catalog manager: success
08:07:17.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:17.009739 7560 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 23f40fe1eb024882a90852518d52efa9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:07:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:17.290239 7584 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:17.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:17.290977 7584 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:17.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:17.292119 7584 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:17.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:17.367290 7584 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:17.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:17.368634 7584 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.30.129
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:17.439553 7584 tablet_server_runner.cc:78] Tablet server non-default flags:
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.30.148:41025
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/data
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/wal
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.30.129:0
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/data/info.pb
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.30.129
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.30.190:37805,127.7.30.189:37049,127.7.30.188:43913
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.30.129
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/logs
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Jun 2026 07:43:21 UTC on e7f111948823
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12467
08:07:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:17.441270 7584 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:17.443852 7584 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:17.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:17.486666 7597 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:17.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:17.713025 7589 catalog_manager.cc:1294] Loaded cluster ID: dda38a83a38b482bb2076f61a3fa9bd2
08:07:17.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:17.714254 7589 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03: loading cluster ID for follower catalog manager: success
08:07:17.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:17.740887 7589 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03: acquiring CA information for follower catalog manager: success
08:07:17.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:17.761092 7589 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 21239e92f47d4e93945ea8a144115d03: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:07:19.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:18.873396 7596 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 7584
08:07:19.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:19.456167 7596 kernel_stack_watchdog.cc:198] Thread 7584 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 382ms:
08:07:19.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:07:19.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:07:19.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:07:19.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:07:19.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:17.494539 7598 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:19.457495 7584 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.969s user 0.599s sys 1.161s
08:07:19.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:19.457828 7584 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.969s user 0.599s sys 1.161s
08:07:19.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.465248 7584 server_base.cc:1061] running on GCE node
08:07:19.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:19.466228 7604 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:19.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.467706 7584 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:19.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.478399 7584 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:19.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.482076 7584 hybrid_clock.cc:648] HybridClock initialized: now 1781856439482006 us; error 55 us; skew 500 ppm
08:07:19.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.483189 7584 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:19.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.494426 7584 webserver.cc:533] Webserver started at http://127.7.30.129:34659/ using document root <none> and password file <none>
08:07:19.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.495492 7584 fs_manager.cc:362] Metadata directory not provided
08:07:19.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.495759 7584 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:19.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.496290 7584 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.503572 7584 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/data/instance:
08:07:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bee9f0405b16404f9051616eb5f89bfa"
08:07:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:19 on dist-test-slave-jt9t"
08:07:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.505244 7584 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/wal/instance:
08:07:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bee9f0405b16404f9051616eb5f89bfa"
08:07:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:19 on dist-test-slave-jt9t"
08:07:19.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.515695 7584 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.006s sys 0.002s
08:07:19.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.523780 7609 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:19.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.525235 7584 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
08:07:19.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.525681 7584 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/data,/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/wal
08:07:19.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bee9f0405b16404f9051616eb5f89bfa"
08:07:19.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:19 on dist-test-slave-jt9t"
08:07:19.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.526201 7584 fs_report.cc:389] FS layout report
08:07:19.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:19.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/wal
08:07:19.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/wal
08:07:19.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/data/data
08:07:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.615407 7584 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:19.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.617465 7584 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:19.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.618079 7584 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:19.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.621714 7584 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:07:19.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.628046 7584 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:07:19.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.628381 7584 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:07:19.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.628727 7584 ts_tablet_manager.cc:616] Registered 0 tablets
08:07:19.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.628961 7584 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:07:19.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.896643 7584 rpc_server.cc:307] RPC server started. Bound to: 127.7.30.129:37345
08:07:19.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.896766 7722 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.30.129:37345 every 8 connection(s)
08:07:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.899529 7584 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/data/info.pb
08:07:19.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.907052 7290 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu as pid 7584
08:07:19.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.907713 7290 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-0/wal/instance
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.925859 7290 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/wal
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/data
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/logs
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/data/info.pb
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:07:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.30.130:0
08:07:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.30.130
08:07:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.30.130
08:07:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.30.190:37805,127.7.30.189:37049,127.7.30.188:43913
08:07:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.30.148:41025
08:07:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:07:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.982266 7723 heartbeater.cc:344] Connected to a master server at 127.7.30.188:43913
08:07:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.991153 7724 heartbeater.cc:344] Connected to a master server at 127.7.30.189:37049
08:07:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.992535 7723 heartbeater.cc:461] Registering TS with master...
08:07:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.992609 7724 heartbeater.cc:461] Registering TS with master...
08:07:19.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.994179 7724 heartbeater.cc:507] Master 127.7.30.189:37049 requested a full tablet report, sending...
08:07:19.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.994218 7723 heartbeater.cc:507] Master 127.7.30.188:43913 requested a full tablet report, sending...
08:07:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.996816 7728 heartbeater.cc:344] Connected to a master server at 127.7.30.190:37805
08:07:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.997223 7728 heartbeater.cc:461] Registering TS with master...
08:07:20.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.998320 7728 heartbeater.cc:507] Master 127.7.30.190:37805 requested a full tablet report, sending...
08:07:20.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.999125 7490 ts_manager.cc:194] Registered new tserver with Master: bee9f0405b16404f9051616eb5f89bfa (127.7.30.129:37345)
08:07:20.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:19.999198 7410 ts_manager.cc:194] Registered new tserver with Master: bee9f0405b16404f9051616eb5f89bfa (127.7.30.129:37345)
08:07:20.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:20.001859 7490 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.7.30.129:53301
08:07:20.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:20.001956 7335 ts_manager.cc:194] Registered new tserver with Master: bee9f0405b16404f9051616eb5f89bfa (127.7.30.129:37345)
08:07:20.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:20.337764 7727 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:20.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:20.338428 7727 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:20.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:20.338971 7727 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:20.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:20.375031 7727 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:20.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:20.376243 7727 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.30.130
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:20.415197 7727 tablet_server_runner.cc:78] Tablet server non-default flags:
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.30.148:41025
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/data
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/wal
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.30.130:0
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/data/info.pb
08:07:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.30.130
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.30.190:37805,127.7.30.189:37049,127.7.30.188:43913
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.30.130
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/logs
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Jun 2026 07:43:21 UTC on e7f111948823
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12467
08:07:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:20.416757 7727 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:20.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:20.418485 7727 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:20.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:20.433301 7739 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:21.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:21.005712 7723 heartbeater.cc:499] Master 127.7.30.188:43913 was elected leader, sending a full tablet report...
08:07:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:21.836506 7738 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 7727
08:07:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:22.437467 7727 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.001s user 0.662s sys 1.139s
08:07:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:22.438843 7727 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.003s user 0.662s sys 1.139s
08:07:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:20.436447 7740 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:22.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:22.440557 7742 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:22.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:22.444373 7741 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2003 milliseconds
08:07:22.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.444412 7727 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:22.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.446051 7727 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.448568 7727 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:22.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.452124 7727 hybrid_clock.cc:648] HybridClock initialized: now 1781856442451786 us; error 329 us; skew 500 ppm
08:07:22.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.453234 7727 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:22.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.461138 7727 webserver.cc:533] Webserver started at http://127.7.30.130:41283/ using document root <none> and password file <none>
08:07:22.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.462410 7727 fs_manager.cc:362] Metadata directory not provided
08:07:22.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.462702 7727 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:22.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.463229 7727 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:22.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.469827 7727 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/data/instance:
08:07:22.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a04398027504a3bbce5c9140cd1636b"
08:07:22.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:22 on dist-test-slave-jt9t"
08:07:22.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.471364 7727 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/wal/instance:
08:07:22.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a04398027504a3bbce5c9140cd1636b"
08:07:22.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:22 on dist-test-slave-jt9t"
08:07:22.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.480609 7727 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.009s sys 0.000s
08:07:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.486358 7752 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.487493 7727 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
08:07:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.487859 7727 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/data,/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/wal
08:07:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a04398027504a3bbce5c9140cd1636b"
08:07:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:22 on dist-test-slave-jt9t"
08:07:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.488224 7727 fs_report.cc:389] FS layout report
08:07:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/wal
08:07:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/wal
08:07:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/data/data
08:07:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:22.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:22.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.541150 7727 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:22.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.542898 7727 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:22.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.543417 7727 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:22.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.546402 7727 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:07:22.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.550819 7727 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:07:22.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.551084 7727 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:07:22.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.551412 7727 ts_tablet_manager.cc:616] Registered 0 tablets
08:07:22.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.551595 7727 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:07:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.699461 7727 rpc_server.cc:307] RPC server started. Bound to: 127.7.30.130:41715
08:07:22.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.699623 7864 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.30.130:41715 every 8 connection(s)
08:07:22.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.703130 7727 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/data/info.pb
08:07:22.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.713056 7290 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu as pid 7727
08:07:22.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.713589 7290 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-1/wal/instance
08:07:22.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.725895 7290 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:22.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/wal
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/data
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/logs
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/data/info.pb
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.30.131:0
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.30.131
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.30.131
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.30.190:37805,127.7.30.189:37049,127.7.30.188:43913
08:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.30.148:41025
08:07:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:07:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.752234 7866 heartbeater.cc:344] Connected to a master server at 127.7.30.189:37049
08:07:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.752794 7866 heartbeater.cc:461] Registering TS with master...
08:07:22.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.754262 7866 heartbeater.cc:507] Master 127.7.30.189:37049 requested a full tablet report, sending...
08:07:22.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.757678 7410 ts_manager.cc:194] Registered new tserver with Master: 7a04398027504a3bbce5c9140cd1636b (127.7.30.130:41715)
08:07:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.759871 7867 heartbeater.cc:344] Connected to a master server at 127.7.30.190:37805
08:07:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.760205 7867 heartbeater.cc:461] Registering TS with master...
08:07:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.760987 7865 heartbeater.cc:344] Connected to a master server at 127.7.30.188:43913
08:07:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.761402 7865 heartbeater.cc:461] Registering TS with master...
08:07:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.761544 7867 heartbeater.cc:507] Master 127.7.30.190:37805 requested a full tablet report, sending...
08:07:22.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.762297 7865 heartbeater.cc:507] Master 127.7.30.188:43913 requested a full tablet report, sending...
08:07:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.764514 7490 ts_manager.cc:194] Registered new tserver with Master: 7a04398027504a3bbce5c9140cd1636b (127.7.30.130:41715)
08:07:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.766245 7490 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.7.30.130:55401
08:07:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:22.766984 7335 ts_manager.cc:194] Registered new tserver with Master: 7a04398027504a3bbce5c9140cd1636b (127.7.30.130:41715)
08:07:23.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:23.067708 7875 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:23.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:23.068297 7875 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:23.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:23.068842 7875 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:23.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:23.102424 7875 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:23.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:23.103350 7875 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.7.30.131
08:07:23.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:23.141448 7875 tablet_server_runner.cc:78] Tablet server non-default flags:
08:07:23.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.30.148:41025
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/data
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/wal
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.30.131:0
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/data/info.pb
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.30.131
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.7.30.190:37805,127.7.30.189:37049,127.7.30.188:43913
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.7.30.131
08:07:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/logs
08:07:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:07:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:07:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Jun 2026 07:43:21 UTC on e7f111948823
08:07:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12467
08:07:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:23.143126 7875 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:23.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:23.144918 7875 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:23.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:23.158126 7881 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:23.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:23.770380 7865 heartbeater.cc:499] Master 127.7.30.188:43913 was elected leader, sending a full tablet report...
08:07:24.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:24.562863 7880 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 7875
08:07:24.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:24.957028 7875 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.799s user 0.664s sys 1.117s
08:07:24.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:24.957506 7875 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.800s user 0.664s sys 1.117s
08:07:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:23.158670 7882 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:24.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:24.959551 7884 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:24.962536 7883 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1800 milliseconds
08:07:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.962543 7875 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:24.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.963824 7875 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:24.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.965828 7875 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:24.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.967242 7875 hybrid_clock.cc:648] HybridClock initialized: now 1781856444967216 us; error 42 us; skew 500 ppm
08:07:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.968017 7875 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:24.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.974548 7875 webserver.cc:533] Webserver started at http://127.7.30.131:39035/ using document root <none> and password file <none>
08:07:24.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.975625 7875 fs_manager.cc:362] Metadata directory not provided
08:07:24.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.975831 7875 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:24.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.976226 7875 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.980772 7875 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/data/instance:
08:07:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "083d9708b0274e6aaeaf2c3efce61785"
08:07:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:24 on dist-test-slave-jt9t"
08:07:24.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.981927 7875 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/wal/instance:
08:07:24.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "083d9708b0274e6aaeaf2c3efce61785"
08:07:24.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:24 on dist-test-slave-jt9t"
08:07:24.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.989284 7875 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
08:07:24.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.996279 7893 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.997542 7875 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.002s
08:07:24.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.998009 7875 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/data,/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/wal
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "083d9708b0274e6aaeaf2c3efce61785"
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:24 on dist-test-slave-jt9t"
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:24.998381 7875 fs_report.cc:389] FS layout report
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/wal
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/wal
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/data/data
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.068054 7875 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:25.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.069675 7875 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:25.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.070297 7875 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:25.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.073062 7875 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:07:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.077693 7875 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:07:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.078003 7875 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
08:07:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.078225 7875 ts_tablet_manager.cc:616] Registered 0 tablets
08:07:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.078362 7875 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:07:25.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.216900 7875 rpc_server.cc:307] RPC server started. Bound to: 127.7.30.131:41833
08:07:25.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.217015 8005 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.30.131:41833 every 8 connection(s)
08:07:25.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.219506 7875 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/data/info.pb
08:07:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.221537 7290 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu as pid 7875
08:07:25.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.222054 7290 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster1160638430728450889/ts-2/wal/instance
08:07:25.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.268831 8006 heartbeater.cc:344] Connected to a master server at 127.7.30.188:43913
08:07:25.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.269433 8006 heartbeater.cc:461] Registering TS with master...
08:07:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.271164 8006 heartbeater.cc:507] Master 127.7.30.188:43913 requested a full tablet report, sending...
08:07:25.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.274192 7490 ts_manager.cc:194] Registered new tserver with Master: 083d9708b0274e6aaeaf2c3efce61785 (127.7.30.131:41833)
08:07:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.276108 7490 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.7.30.131:48071
08:07:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.277078 8008 heartbeater.cc:344] Connected to a master server at 127.7.30.190:37805
08:07:25.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.277894 8008 heartbeater.cc:461] Registering TS with master...
08:07:25.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.278872 8008 heartbeater.cc:507] Master 127.7.30.190:37805 requested a full tablet report, sending...
08:07:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.282380 7335 ts_manager.cc:194] Registered new tserver with Master: 083d9708b0274e6aaeaf2c3efce61785 (127.7.30.131:41833)
08:07:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.282495 8007 heartbeater.cc:344] Connected to a master server at 127.7.30.189:37049
08:07:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.282802 8007 heartbeater.cc:461] Registering TS with master...
08:07:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.283442 8007 heartbeater.cc:507] Master 127.7.30.189:37049 requested a full tablet report, sending...
08:07:25.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.285674 7410 ts_manager.cc:194] Registered new tserver with Master: 083d9708b0274e6aaeaf2c3efce61785 (127.7.30.131:41833)
08:07:25.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.290215 7290 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
08:07:25.301 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:07:25.301 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:07:25.302 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.30.190"
port: 37805
}
bound_http_address {
host: "127.7.30.190"
port: 46715
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.30.189"
port: 37049
}
bound_http_address {
host: "127.7.30.189"
port: 36423
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.30.188"
port: 43913
}
bound_http_address {
host: "127.7.30.188"
port: 40917
}
}
}
08:07:25.303 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:07:25.304 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.7.30.129"
port: 37345
}
bound_http_address {
host: "127.7.30.129"
port: 34659
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.7.30.130"
port: 41715
}
bound_http_address {
host: "127.7.30.130"
port: 41283
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.7.30.131"
port: 41833
}
bound_http_address {
host: "127.7.30.131"
port: 39035
}
}
}
08:07:25.304 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:07:25.305 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster11238396738873614621/krb5kdc/krb5cc
08:07:25.306 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:07:25.306 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskfpJR7l/test-tmp
08:07:25.307 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:07:25.307 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskfpJR7l/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:07:25.318 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
08:07:25.659 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:25.659 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:07:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.662982 8017 mini_oidc.cc:150] Starting JWKS server
08:07:25.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.666468 8017 webserver.cc:533] Webserver started at http://127.0.0.1:41527/ using document root <none> and password file <none>
08:07:25.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.667011 8017 mini_oidc.cc:186] Starting OIDC Discovery server
08:07:25.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:25.668769 8017 webserver.cc:533] Webserver started at http://127.0.0.1:39847/ using document root <none> and password file <none>
08:07:25.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:07:25.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/principal' for realm 'KRBTEST.COM',
08:07:25.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:07:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:25 dist-test-slave-jt9t krb5kdc[8028](info): setting up network...
08:07:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:07:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:25 dist-test-slave-jt9t krb5kdc[8028](info): set up 2 sockets
08:07:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:25 dist-test-slave-jt9t krb5kdc[8028](info): commencing operation
08:07:25.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:07:26.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:26.281091 8006 heartbeater.cc:499] Master 127.7.30.188:43913 was elected leader, sending a full tablet report...
08:07:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:27.806660 8017 mini_kdc.cc:121] Time spent starting KDC: real 2.099s user 0.002s sys 0.018s
08:07:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:07:27.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:07:27.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:07:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:07:27.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:07:27.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:07:27.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:07:27.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:07:27.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:07:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:07:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/test-user.keytab.
08:07:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/test-user.keytab.
08:07:27.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:27 dist-test-slave-jt9t krb5kdc[8028](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856447, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:27.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:07:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T08:07:27Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:07:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T08:07:27Z Disabled control of system clock
08:07:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.212.126@KRBTEST.COM; defaulting to no policy
08:07:27.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:27.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.212.126@KRBTEST.COM" created.
08:07:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.212.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab.
08:07:28.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.212.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab.
08:07:28.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.7.212.126@KRBTEST.COM; defaulting to no policy
08:07:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.7.212.126@KRBTEST.COM" created.
08:07:28.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:28.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.7.212.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab.
08:07:28.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.7.212.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab.
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:28.072714 8017 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/logs
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data/info.pb
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.212.126:45529
08:07:28.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.212.126
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.212.126
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.212.84:41237
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.212.126:45529,127.7.212.125:41383,127.7.212.124:35659
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41527/jwks.json
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:28.415401 8045 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:28.415962 8045 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:28.416549 8045 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:28.452005 8045 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:28.452303 8045 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:07:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:28.452783 8045 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:28.453107 8045 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:28.453436 8045 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:07:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:28.453742 8045 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:28.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:28.454015 8045 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41527/jwks.json
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:28.495893 8045 master_runner.cc:387] Master server non-default flags:
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.212.84:41237
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.212.126:45529,127.7.212.125:41383,127.7.212.124:35659
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.212.126
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.212.126:45529
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41527/jwks.json
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data/info.pb
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.212.126
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/logs
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Jun 2026 07:43:21 UTC on e7f111948823
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12467
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:28.497550 8045 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:28.498701 8045 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:28.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:28.511087 8051 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:28.512132 8052 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:29.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:29.973875 8054 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:29.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:29.976567 8053 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1459 milliseconds
08:07:29.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:29.976645 8045 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:29.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:29.978026 8045 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:29.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:29.981113 8045 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:29.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:29.982574 8045 hybrid_clock.cc:648] HybridClock initialized: now 1781856449982522 us; error 61 us; skew 500 ppm
08:07:29.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:29.983573 8045 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:29 dist-test-slave-jt9t krb5kdc[8028](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856449, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:29.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:29.989665 8045 init.cc:377] Logged in from keytab as kudu/127.7.212.126@KRBTEST.COM (short username kudu)
08:07:29.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:29.995520 8045 webserver.cc:533] Webserver started at http://127.7.212.126:33313/ using document root <none> and password file <none>
08:07:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:29.996438 8045 fs_manager.cc:362] Metadata directory not provided
08:07:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:29.996641 8045 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:29.997047 8045 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:30.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.001547 8045 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data/instance:
08:07:30.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "618d653260f5431690fea318de34838f"
08:07:30.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:29 on dist-test-slave-jt9t"
08:07:30.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.002714 8045 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal/instance:
08:07:30.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "618d653260f5431690fea318de34838f"
08:07:30.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:29 on dist-test-slave-jt9t"
08:07:30.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.010568 8045 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
08:07:30.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.016968 8065 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:30.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.018318 8045 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.018656 8045 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data,/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "618d653260f5431690fea318de34838f"
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:29 on dist-test-slave-jt9t"
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.019033 8045 fs_report.cc:389] FS layout report
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data/data
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:30.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.093214 8045 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.098945 8045 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.099360 8045 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:30.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.178010 8045 rpc_server.cc:307] RPC server started. Bound to: 127.7.212.126:45529
08:07:30.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.178133 8116 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.212.126:45529 every 8 connection(s)
08:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.180711 8045 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data/info.pb
08:07:30.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.186741 8117 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:30.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.188462 8017 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu as pid 8045
08:07:30.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.188856 8017 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal/instance
08:07:30.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.199028 8117 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } has no permanent_uuid. Determining permanent_uuid...
08:07:30.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.212.125@KRBTEST.COM; defaulting to no policy
08:07:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:30.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.212.125@KRBTEST.COM" created.
08:07:30.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:30 dist-test-slave-jt9t krb5kdc[8028](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856449, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.126@KRBTEST.COM for kudu/127.7.212.126@KRBTEST.COM
08:07:30.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:30.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.212.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab.
08:07:30.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.212.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab.
08:07:30.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.243287 8117 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } has no permanent_uuid. Determining permanent_uuid...
08:07:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.246361 8066 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:30.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.251034 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.125:41383: Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:30.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.7.212.125@KRBTEST.COM; defaulting to no policy
08:07:30.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:30.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.7.212.125@KRBTEST.COM" created.
08:07:30.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.7.212.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab.
08:07:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.7.212.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab.
08:07:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.287909 8017 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal
08:07:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data
08:07:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/logs
08:07:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data/info.pb
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.212.125:41383
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.212.125
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.212.125
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.212.84:41237
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.212.126:45529,127.7.212.125:41383,127.7.212.124:35659
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41527/jwks.json
08:07:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:30.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:30.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.301079 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } attempt: 1
08:07:30.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.307433 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.125:41383: Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:30.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.375881 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } attempt: 2
08:07:30.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.381215 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.125:41383: Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:30.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.472697 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } attempt: 3
08:07:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.477782 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.125:41383: Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:30.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.621315 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } attempt: 4
08:07:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.626035 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.125:41383: Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.636191 8126 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:30.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.636871 8126 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:30.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.637593 8126 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:30.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.674604 8126 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:30.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.674919 8126 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:07:30.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.675410 8126 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:30.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.675757 8126 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:30.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.676085 8126 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:07:30.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.676414 8126 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:30.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.676664 8126 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41527/jwks.json
08:07:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.725236 8126 master_runner.cc:387] Master server non-default flags:
08:07:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.212.84:41237
08:07:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data
08:07:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal
08:07:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.212.126:45529,127.7.212.125:41383,127.7.212.124:35659
08:07:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab
08:07:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.212.125
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.212.125:41383
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41527/jwks.json
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data/info.pb
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.212.125
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/logs
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Jun 2026 07:43:21 UTC on e7f111948823
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12467
08:07:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:30.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.726953 8126 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.728053 8126 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:30.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.739625 8133 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:30.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:30.925424 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } attempt: 5
08:07:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.933389 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.125:41383: Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:31.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:31.481112 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } attempt: 6
08:07:31.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:31.492543 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.125:41383: Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:32.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.746354 8136 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:32.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:30.742267 8134 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:32.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.114663 8135 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1368 milliseconds
08:07:32.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.114842 8126 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:32.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.116228 8126 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:32.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.119805 8126 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:32.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.121286 8126 hybrid_clock.cc:648] HybridClock initialized: now 1781856452121237 us; error 69 us; skew 500 ppm
08:07:32.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.122195 8126 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:32.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:32 dist-test-slave-jt9t krb5kdc[8028](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856452, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:32.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.128479 8126 init.cc:377] Logged in from keytab as kudu/127.7.212.125@KRBTEST.COM (short username kudu)
08:07:32.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.133915 8126 webserver.cc:533] Webserver started at http://127.7.212.125:33749/ using document root <none> and password file <none>
08:07:32.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.134914 8126 fs_manager.cc:362] Metadata directory not provided
08:07:32.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.135121 8126 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:32.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.135527 8126 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:32.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.140146 8126 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data/instance:
08:07:32.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c450723ed6be41008af218d230566e04"
08:07:32.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:32 on dist-test-slave-jt9t"
08:07:32.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.141266 8126 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal/instance:
08:07:32.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c450723ed6be41008af218d230566e04"
08:07:32.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:32 on dist-test-slave-jt9t"
08:07:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.149936 8126 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.011s sys 0.000s
08:07:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.155938 8147 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.157177 8126 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
08:07:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.157609 8126 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data,/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal
08:07:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c450723ed6be41008af218d230566e04"
08:07:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:32 on dist-test-slave-jt9t"
08:07:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.158190 8126 fs_report.cc:389] FS layout report
08:07:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal
08:07:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal
08:07:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data/data
08:07:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.222260 8126 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:32.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.228447 8126 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:32.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.229004 8126 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.308048 8126 rpc_server.cc:307] RPC server started. Bound to: 127.7.212.125:41383
08:07:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.308305 8198 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.212.125:41383 every 8 connection(s)
08:07:32.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.310809 8126 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data/info.pb
08:07:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.311908 8017 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu as pid 8126
08:07:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.312376 8017 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal/instance
08:07:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.320387 8199 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:32.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.212.124@KRBTEST.COM; defaulting to no policy
08:07:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.212.124@KRBTEST.COM" created.
08:07:32.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.334617 8199 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } has no permanent_uuid. Determining permanent_uuid...
08:07:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:32 dist-test-slave-jt9t krb5kdc[8028](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856452, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.125@KRBTEST.COM for kudu/127.7.212.126@KRBTEST.COM
08:07:32.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:32.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.212.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab.
08:07:32.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.212.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab.
08:07:32.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.375186 8199 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } has no permanent_uuid. Determining permanent_uuid...
08:07:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.7.212.124@KRBTEST.COM; defaulting to no policy
08:07:32.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:32.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.7.212.124@KRBTEST.COM" created.
08:07:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:32 dist-test-slave-jt9t krb5kdc[8028](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856452, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.125@KRBTEST.COM for kudu/127.7.212.125@KRBTEST.COM
08:07:32.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.407392 8199 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } has no permanent_uuid. Determining permanent_uuid...
08:07:32.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.410418 8149 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:32.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:07:32.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.7.212.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab.
08:07:32.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.7.212.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab.
08:07:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.414693 8199 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.415300 8017 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/wal
08:07:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data
08:07:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/logs
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data/info.pb
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.212.124:35659
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.212.124
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.212.124
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.212.84:41237
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.212.126:45529,127.7.212.125:41383,127.7.212.124:35659
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41527/jwks.json
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:32.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.464267 8199 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 1
08:07:32.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.468997 8199 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.537511 8199 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 2
08:07:32.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.542168 8199 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:32.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.553165 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } attempt: 7
08:07:32.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:32 dist-test-slave-jt9t krb5kdc[8028](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856449, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.126@KRBTEST.COM for kudu/127.7.212.125@KRBTEST.COM
08:07:32.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.579495 8117 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } has no permanent_uuid. Determining permanent_uuid...
08:07:32.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.587697 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:32.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.633684 8199 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 3
08:07:32.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.639365 8199 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:32.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.646150 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 1
08:07:32.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.651099 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.732585 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 2
08:07:32.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.738193 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:32.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.758193 8209 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:32.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.758809 8209 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:32.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.759420 8209 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:32.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.782888 8199 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 4
08:07:32.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.787508 8199 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:32.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.795750 8209 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:32.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.796001 8209 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:07:32.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.796264 8209 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:32.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.796474 8209 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:32.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.796687 8209 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:07:32.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.796878 8209 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:32.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.797029 8209 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41527/jwks.json
08:07:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.823665 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 3
08:07:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.828120 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.840749 8209 master_runner.cc:387] Master server non-default flags:
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.212.84:41237
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/wal
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.212.126:45529,127.7.212.125:41383,127.7.212.124:35659
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.212.124
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.212.124:35659
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41527/jwks.json
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data/info.pb
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.212.124
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/logs
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Jun 2026 07:43:21 UTC on e7f111948823
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12467
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:32.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.842227 8209 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:32.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.843318 8209 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:32.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.856436 8218 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:32.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:32.968603 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 4
08:07:32.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.989494 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:33.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:33.086920 8199 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 5
08:07:33.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:33.095741 8199 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:33.273159 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 5
08:07:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:33.283111 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:33.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:33.643311 8199 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 6
08:07:33.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:33.690076 8199 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:33.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:33.835930 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 6
08:07:33.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:33.847041 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:34.258726 8217 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 8209
08:07:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:34.719573 8217 kernel_stack_watchdog.cc:198] Thread 8209 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
08:07:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:07:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:07:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:07:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:07:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:32.856695 8219 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:34.721473 8209 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.865s user 0.615s sys 1.212s
08:07:34.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:34.721876 8209 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.866s user 0.615s sys 1.212s
08:07:34.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:34.722530 8221 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:34.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:34.728791 8220 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1871 milliseconds
08:07:34.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.728819 8209 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:34.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.730295 8209 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:34.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.732856 8209 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:34.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.734331 8209 hybrid_clock.cc:648] HybridClock initialized: now 1781856454734283 us; error 40 us; skew 500 ppm
08:07:34.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.735249 8209 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:34.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:34 dist-test-slave-jt9t krb5kdc[8028](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856454, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:34.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.741789 8209 init.cc:377] Logged in from keytab as kudu/127.7.212.124@KRBTEST.COM (short username kudu)
08:07:34.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.746845 8209 webserver.cc:533] Webserver started at http://127.7.212.124:37137/ using document root <none> and password file <none>
08:07:34.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.748153 8209 fs_manager.cc:362] Metadata directory not provided
08:07:34.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.748383 8209 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.748858 8209 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:07:34.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.750672 8199 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 7
08:07:34.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.753720 8209 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data/instance:
08:07:34.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f73f0cebe6f4dd89070e1842d3de8d9"
08:07:34.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:34 on dist-test-slave-jt9t"
08:07:34.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.755466 8209 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/wal/instance:
08:07:34.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f73f0cebe6f4dd89070e1842d3de8d9"
08:07:34.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:34 on dist-test-slave-jt9t"
08:07:34.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:34.758994 8199 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:34.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.764629 8209 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
08:07:34.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.770732 8237 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.772132 8209 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
08:07:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.772439 8209 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data,/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/wal
08:07:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f73f0cebe6f4dd89070e1842d3de8d9"
08:07:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:34 on dist-test-slave-jt9t"
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.772784 8209 fs_report.cc:389] FS layout report
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/wal
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/wal
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data/data
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:34.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.831487 8209 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:34.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.844007 8209 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:34.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.844614 8209 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.880657 8117 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } attempt: 7
08:07:34.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:34.888310 8117 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.7.212.124:35659: Network error: Client connection negotiation failed: client connection to 127.7.212.124:35659: connect: Connection refused (error 111)
08:07:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.927579 8209 rpc_server.cc:307] RPC server started. Bound to: 127.7.212.124:35659
08:07:34.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.927657 8290 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.212.124:35659 every 8 connection(s)
08:07:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.930905 8209 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data/info.pb
08:07:34.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.937907 8291 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:07:34.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.939153 8017 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu as pid 8209
08:07:34.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.939641 8017 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/wal/instance
08:07:34.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.951494 8291 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } has no permanent_uuid. Determining permanent_uuid...
08:07:34.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:34 dist-test-slave-jt9t krb5kdc[8028](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856454, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.124@KRBTEST.COM for kudu/127.7.212.126@KRBTEST.COM
08:07:34.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:34 dist-test-slave-jt9t krb5kdc[8028](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856447, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.212.124@KRBTEST.COM
08:07:34.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:34.986439 8291 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } has no permanent_uuid. Determining permanent_uuid...
08:07:34.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:34 dist-test-slave-jt9t krb5kdc[8028](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856454, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.124@KRBTEST.COM for kudu/127.7.212.125@KRBTEST.COM
08:07:35.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:34 dist-test-slave-jt9t krb5kdc[8028](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856447, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.212.125@KRBTEST.COM
08:07:35.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.011161 8291 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } has no permanent_uuid. Determining permanent_uuid...
08:07:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:35 dist-test-slave-jt9t krb5kdc[8028](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856454, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.124@KRBTEST.COM for kudu/127.7.212.124@KRBTEST.COM
08:07:35.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:35 dist-test-slave-jt9t krb5kdc[8028](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856447, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.212.126@KRBTEST.COM
08:07:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.041743 8291 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: Bootstrap starting.
08:07:35.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.048699 8017 external_mini_cluster.cc:955] 0 TS(s) registered with all masters
08:07:35.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.049042 8291 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: Neither blocks nor log segments found. Creating new log.
08:07:35.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.050956 8291 log.cc:826] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: Log is configured to *not* fsync() on all Append() calls
08:07:35.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.055673 8291 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: No bootstrap required, opened a new log
08:07:35.057 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:35.057 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
08:07:35.058 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
08:07:35.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
08:07:35.060 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.212.126"
port: 45529
}
bound_http_address {
host: "127.7.212.126"
port: 33313
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.212.125"
port: 41383
}
bound_http_address {
host: "127.7.212.125"
port: 33749
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.212.124"
port: 35659
}
bound_http_address {
host: "127.7.212.124"
port: 37137
}
}
}
08:07:35.060 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:07:35.060 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:35.060 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:07:35.060 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:07:35.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.081411 8291 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "618d653260f5431690fea318de34838f" member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } } peers { permanent_uuid: "c450723ed6be41008af218d230566e04" member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } } peers { permanent_uuid: "3f73f0cebe6f4dd89070e1842d3de8d9" member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } }
08:07:35.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.082161 8291 raft_consensus.cc:385] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:35.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.082469 8291 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3f73f0cebe6f4dd89070e1842d3de8d9, State: Initialized, Role: FOLLOWER
08:07:35.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.083297 8291 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "618d653260f5431690fea318de34838f" member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } } peers { permanent_uuid: "c450723ed6be41008af218d230566e04" member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } } peers { permanent_uuid: "3f73f0cebe6f4dd89070e1842d3de8d9" member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } }
08:07:35.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.086223 8302 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "618d653260f5431690fea318de34838f" member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } } peers { permanent_uuid: "c450723ed6be41008af218d230566e04" member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } } peers { permanent_uuid: "3f73f0cebe6f4dd89070e1842d3de8d9" member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } } }
08:07:35.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.087317 8302 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [sys.catalog]: This master's current role is: FOLLOWER
08:07:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.088160 8291 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [sys.catalog]: configured and running, proceeding with master startup.
08:07:35.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:35 dist-test-slave-jt9t krb5kdc[8028](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856455, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:35.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:07:35.100 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:35.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:35.100232 8243 tablet.cc:2406] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:07:35.103 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5cc
08:07:35.104 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:07:35.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:35.119089 8313 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:35.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:35.119410 8313 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:35.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.132849 8302 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:35.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.133414 8302 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "618d653260f5431690fea318de34838f" member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } } peers { permanent_uuid: "c450723ed6be41008af218d230566e04" member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } } peers { permanent_uuid: "3f73f0cebe6f4dd89070e1842d3de8d9" member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } }
08:07:35.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.135952 8302 leader_election.cc:290] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 618d653260f5431690fea318de34838f (127.7.212.126:45529), c450723ed6be41008af218d230566e04 (127.7.212.125:41383)
08:07:35.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.116083 8209 master_runner.cc:187] Error getting master registration for 127.7.212.126:45529: OK, instance_id { permanent_uuid: "618d653260f5431690fea318de34838f" instance_seqno: 1781856450155475 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:07:35.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:35.147511 8209 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:07:36.105 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.212.125:41383
08:07:36.105 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:07:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:36.106642 8017 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu with pid 8126
08:07:36.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.124528 8313 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.124902 8313 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:36.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.129894 8238 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.7.212.125:41383 (error 108)
08:07:36.140 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:36.140 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.212.126:45529
08:07:36.141 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:07:36.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:36.141379 8017 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu with pid 8045
08:07:36.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.145875 8238 leader_election.cc:336] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c450723ed6be41008af218d230566e04 (127.7.212.125:41383): Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:36.164 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:36.164 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.212.124:35659
08:07:36.165 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:07:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:36.166052 8017 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu with pid 8209
08:07:36.189 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:36.189 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.212.125:41383
08:07:36.190 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:36.191627 8017 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/logs
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data/info.pb
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.212.125:41383
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.212.125
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33749
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.212.125
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.212.84:41237
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.212.126:45529,127.7.212.125:41383,127.7.212.124:35659
08:07:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41527/jwks.json
08:07:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:36.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:36.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.527443 8318 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:36.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.528021 8318 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:36.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.528798 8318 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:36.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.567286 8318 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:36.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.567525 8318 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:07:36.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.567791 8318 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.568003 8318 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.568207 8318 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:07:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.568400 8318 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.568560 8318 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41527/jwks.json
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:36.609966 8318 master_runner.cc:387] Master server non-default flags:
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.212.84:41237
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.212.126:45529,127.7.212.125:41383,127.7.212.124:35659
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.212.125
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.212.125:41383
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41527/jwks.json
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:36.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data/info.pb
08:07:36.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.212.125
08:07:36.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33749
08:07:36.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:36.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/logs
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Jun 2026 07:43:21 UTC on e7f111948823
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12467
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:36.611603 8318 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:36.612845 8318 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:36.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.624719 8324 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:37.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.625674 8325 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:37.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:36.629146 8327 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:37.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:37.935974 8326 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1305 milliseconds
08:07:37.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.936146 8318 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:37.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.937399 8318 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.940619 8318 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.942108 8318 hybrid_clock.cc:648] HybridClock initialized: now 1781856457942029 us; error 80 us; skew 500 ppm
08:07:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.942961 8318 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:37.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:37 dist-test-slave-jt9t krb5kdc[8028](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856457, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.949259 8318 init.cc:377] Logged in from keytab as kudu/127.7.212.125@KRBTEST.COM (short username kudu)
08:07:37.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.954785 8318 webserver.cc:533] Webserver started at http://127.7.212.125:33749/ using document root <none> and password file <none>
08:07:37.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.955708 8318 fs_manager.cc:362] Metadata directory not provided
08:07:37.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.955936 8318 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:37.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.964006 8318 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
08:07:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.968552 8337 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:37.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.969715 8318 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.970130 8318 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data,/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c450723ed6be41008af218d230566e04"
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:32 on dist-test-slave-jt9t"
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:37.972163 8318 fs_report.cc:389] FS layout report
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data/data
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.038786 8318 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:38.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.045279 8318 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:38.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.045873 8318 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:38.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.120817 8318 rpc_server.cc:307] RPC server started. Bound to: 127.7.212.125:41383
08:07:38.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.120911 8388 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.212.125:41383 every 8 connection(s)
08:07:38.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.124656 8318 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/data/info.pb
08:07:38.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.126454 8017 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu as pid 8318
08:07:38.127 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:38.127 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.212.126:45529
08:07:38.127 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.129113 8017 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/logs
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data/info.pb
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:38.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.212.126:45529
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.212.126
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33313
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.212.126
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.212.84:41237
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.212.126:45529,127.7.212.125:41383,127.7.212.124:35659
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41527/jwks.json
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:38.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:38.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.137802 8389 sys_catalog.cc:263] Verifying existing consensus state
08:07:38.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260619 08:07:38.138684 8389 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:07:38.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.139124 8318 master.cc:562] Master@127.7.212.125:41383 shutting down...
08:07:38.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.206723 8318 master.cc:584] Master@127.7.212.125:41383 shutdown complete.
08:07:38.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:38.209075 8318 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
08:07:38.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:07:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:38.472177 8391 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:38.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:38.472815 8391 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:38.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:38.473429 8391 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:38.507169 8391 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:38.507403 8391 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:07:38.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:38.507681 8391 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:38.507884 8391 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:38.508091 8391 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:07:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:38.508282 8391 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:38.508428 8391 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41527/jwks.json
08:07:38.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.548473 8391 master_runner.cc:387] Master server non-default flags:
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.212.84:41237
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.212.126:45529,127.7.212.125:41383,127.7.212.124:35659
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.212.126
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.212.126:45529
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41527/jwks.json
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data/info.pb
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.212.126
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33313
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:38.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/logs
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Jun 2026 07:43:21 UTC on e7f111948823
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12467
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.549690 8391 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:38.551090 8391 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:38.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:38.571904 8397 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:39.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:39.969136 8396 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 8391
08:07:39.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:38.573486 8398 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:39.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:39.976945 8391 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.408s user 0.507s sys 0.900s
08:07:39.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:39.977311 8391 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.408s user 0.507s sys 0.900s
08:07:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:39.986229 8403 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:39.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:39.986218 8391 server_base.cc:1061] running on GCE node
08:07:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:39.987644 8391 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:39.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:39.990132 8391 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:39.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:39.991572 8391 hybrid_clock.cc:648] HybridClock initialized: now 1781856459991523 us; error 37 us; skew 500 ppm
08:07:39.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:39.992496 8391 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:39.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:39 dist-test-slave-jt9t krb5kdc[8028](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856459, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:39.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:39.999092 8391 init.cc:377] Logged in from keytab as kudu/127.7.212.126@KRBTEST.COM (short username kudu)
08:07:40.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.005359 8391 webserver.cc:533] Webserver started at http://127.7.212.126:33313/ using document root <none> and password file <none>
08:07:40.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.006485 8391 fs_manager.cc:362] Metadata directory not provided
08:07:40.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.006767 8391 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:40.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.017392 8391 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.010s sys 0.000s
08:07:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.022394 8410 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.024852 8391 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
08:07:40.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.025184 8391 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data,/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal
08:07:40.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "618d653260f5431690fea318de34838f"
08:07:40.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:29 on dist-test-slave-jt9t"
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.027599 8391 fs_report.cc:389] FS layout report
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data/data
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:40.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.092337 8391 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:40.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.098259 8391 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:40.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.098680 8391 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:40.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.175679 8391 rpc_server.cc:307] RPC server started. Bound to: 127.7.212.126:45529
08:07:40.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.175729 8461 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.212.126:45529 every 8 connection(s)
08:07:40.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.178743 8391 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/data/info.pb
08:07:40.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.179514 8017 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu as pid 8391
08:07:40.180 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:40.180 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.212.124:35659
08:07:40.180 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:07:40.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.181483 8017 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:40.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu
08:07:40.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/wal
08:07:40.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data
08:07:40.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:40.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:40.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:40.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:40.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/logs
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data/info.pb
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.212.124:35659
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.212.124
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37137
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.212.124
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.212.84:41237
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.212.126:45529,127.7.212.125:41383,127.7.212.124:35659
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41527/jwks.json
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:07:40.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:07:40.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.193503 8462 sys_catalog.cc:263] Verifying existing consensus state
08:07:40.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260619 08:07:40.194461 8462 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:07:40.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.194944 8391 master.cc:562] Master@127.7.212.126:45529 shutting down...
08:07:40.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.265368 8391 master.cc:584] Master@127.7.212.126:45529 shutdown complete.
08:07:40.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:40.268368 8391 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
08:07:40.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:07:40.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:40.529314 8464 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:40.530010 8464 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:40.530812 8464 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:40.564759 8464 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:07:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:40.565013 8464 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:07:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:40.565274 8464 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:40.565475 8464 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:40.565671 8464 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:07:40.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:40.565939 8464 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:40.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:40.566110 8464 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:41527/jwks.json
08:07:40.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.607956 8464 master_runner.cc:387] Master server non-default flags:
08:07:40.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:40.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.212.84:41237
08:07:40.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:40.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:40.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data
08:07:40.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/wal
08:07:40.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:07:40.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.212.126:45529,127.7.212.125:41383,127.7.212.124:35659
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/kudu.keytab
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.212.124
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.212.124:35659
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41527/jwks.json
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data/info.pb
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.212.124
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37137
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/logs
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Jun 2026 07:43:21 UTC on e7f111948823
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12467
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:40.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.609298 8464 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:40.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:40.610577 8464 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:40.623842 8471 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:42.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:42.028741 8470 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 8464
08:07:42.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:40.624711 8472 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:42.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:42.308115 8470 kernel_stack_watchdog.cc:198] Thread 8464 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
08:07:42.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:07:42.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:07:42.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:07:42.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:07:42.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:42.308408 8464 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.685s user 0.562s sys 1.008s
08:07:42.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:42.309001 8464 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.686s user 0.562s sys 1.008s
08:07:42.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:42.310285 8474 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:07:42.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:42.312809 8473 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1682 milliseconds
08:07:42.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.312915 8464 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:07:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.314385 8464 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:07:42.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.316902 8464 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:07:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.318337 8464 hybrid_clock.cc:648] HybridClock initialized: now 1781856462318287 us; error 46 us; skew 500 ppm
08:07:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.319329 8464 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:07:42.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:42 dist-test-slave-jt9t krb5kdc[8028](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856462, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:42.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.325778 8464 init.cc:377] Logged in from keytab as kudu/127.7.212.124@KRBTEST.COM (short username kudu)
08:07:42.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.330904 8464 webserver.cc:533] Webserver started at http://127.7.212.124:37137/ using document root <none> and password file <none>
08:07:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.331904 8464 fs_manager.cc:362] Metadata directory not provided
08:07:42.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.332180 8464 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:07:42.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.340853 8464 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
08:07:42.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.345657 8483 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:07:42.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.347223 8464 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.003s
08:07:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.347621 8464 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data,/tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/wal
08:07:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f73f0cebe6f4dd89070e1842d3de8d9"
08:07:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 08:07:34 on dist-test-slave-jt9t"
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.349634 8464 fs_report.cc:389] FS layout report
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/wal
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/wal
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data/data
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:07:42.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:07:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.416277 8464 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:07:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.422144 8464 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:07:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.422709 8464 kserver.cc:163] Server-wide thread pool size limit: 3276
08:07:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.506403 8464 rpc_server.cc:307] RPC server started. Bound to: 127.7.212.124:35659
08:07:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.506501 8534 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.7.212.124:35659 every 8 connection(s)
08:07:42.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.509299 8464 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/master-2/data/info.pb
08:07:42.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.511219 8017 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu as pid 8464
08:07:42.511 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:07:42.513 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:42.515 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:42.516 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.516 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.517 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.517 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.518 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.212.124:35659(127.7.212.124:35659)] connecting to peer
08:07:42.519 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.520 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.521 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.7.212.124:35659(127.7.212.124:35659)] Successfully connected to peer
08:07:42.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.527866 8535 sys_catalog.cc:263] Verifying existing consensus state
08:07:42.534 [DEBUG - kudu-nio-2] (SecurityContext.java:222) Refreshing Kerberos credentials...
08:07:42.534 [DEBUG - kudu-nio-2] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskfpJR7l/test-tmp/mini-kudu-cluster5344757795564942895/krb5kdc/krb5cc
08:07:42.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.537468 8535 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: Bootstrap starting.
08:07:42.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:42 dist-test-slave-jt9t krb5kdc[8028](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781856455, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:42.540 [DEBUG - kudu-nio-2] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:07:42.540 [INFO - kudu-nio-2] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
08:07:42.541 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.7.212.124
08:07:42.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:42 dist-test-slave-jt9t krb5kdc[8028](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781856455, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.212.124@KRBTEST.COM
08:07:42.577 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xcbf5ed35, L:/127.0.0.1:33404 - R:/127.7.212.124:35659] using SASL/GSSAPI
08:07:42.586 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.124:35659: Catalog manager is not initialized. State: Starting
08:07:42.586 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.591817 8535 log.cc:826] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: Log is configured to *not* fsync() on all Append() calls
08:07:42.593 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:42.604 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:42.607 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.608 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.609 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.612 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.124:35659: Catalog manager is not initialized. State: Starting
08:07:42.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.612534 8535 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:07:42.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.613476 8535 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: Bootstrap complete.
08:07:42.614 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.614 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.614 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.614 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:42.633 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:42.635 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:42.636 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.636 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.636 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.635922 8535 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "618d653260f5431690fea318de34838f" member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } } peers { permanent_uuid: "c450723ed6be41008af218d230566e04" member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } } peers { permanent_uuid: "3f73f0cebe6f4dd89070e1842d3de8d9" member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } }
08:07:42.640 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.124:35659: Catalog manager is not initialized. State: Starting
08:07:42.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.639449 8535 raft_consensus.cc:385] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:07:42.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.639916 8535 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3f73f0cebe6f4dd89070e1842d3de8d9, State: Initialized, Role: FOLLOWER
08:07:42.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.641161 8535 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "618d653260f5431690fea318de34838f" member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } } peers { permanent_uuid: "c450723ed6be41008af218d230566e04" member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } } peers { permanent_uuid: "3f73f0cebe6f4dd89070e1842d3de8d9" member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } }
08:07:42.644 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.645 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.646 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.646 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:42.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.647936 8548 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "618d653260f5431690fea318de34838f" member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } } peers { permanent_uuid: "c450723ed6be41008af218d230566e04" member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } } peers { permanent_uuid: "3f73f0cebe6f4dd89070e1842d3de8d9" member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } } }
08:07:42.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.648885 8548 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [sys.catalog]: This master's current role is: FOLLOWER
08:07:42.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.649266 8535 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [sys.catalog]: configured and running, proceeding with master startup.
08:07:42.654 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:42.656 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:42.657 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.657 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.657 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.658594 8548 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:42.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.659308 8548 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "618d653260f5431690fea318de34838f" member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } } peers { permanent_uuid: "c450723ed6be41008af218d230566e04" member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } } peers { permanent_uuid: "3f73f0cebe6f4dd89070e1842d3de8d9" member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } }
08:07:42.663 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.124:35659: Catalog manager is not initialized. State: Starting
08:07:42.664 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.664 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.664 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.665 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:42.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.666209 8548 leader_election.cc:290] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 618d653260f5431690fea318de34838f (127.7.212.126:45529), c450723ed6be41008af218d230566e04 (127.7.212.125:41383)
08:07:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:42.674535 8484 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111)
08:07:42.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:42.678268 8489 tablet.cc:2406] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:07:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:42.682471 8484 leader_election.cc:336] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 618d653260f5431690fea318de34838f (127.7.212.126:45529): Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111)
08:07:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:42.683239 8484 leader_election.cc:336] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c450723ed6be41008af218d230566e04 (127.7.212.125:41383): Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.683861 8484 leader_election.cc:304] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3f73f0cebe6f4dd89070e1842d3de8d9; no voters: 618d653260f5431690fea318de34838f, c450723ed6be41008af218d230566e04
08:07:42.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.684628 8548 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:07:42.693 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:42.695 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:42.695 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.695 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.696 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.696 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.696 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.696 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.699 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:42.705724 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:42.706179 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:42.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.691816 8464 master_runner.cc:187] Error getting master registration for 127.7.212.126:45529: Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111),
08:07:42.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:42.718466 8464 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:07:42.733 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:42.734 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:42.734 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.734 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.735 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.735 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.735 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.735 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.739 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:42.774 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:42.775 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:42.775 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.775 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.775 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.776 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.776 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.776 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.779 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:42.833 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:42.834 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:42.834 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.835 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.835 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:42.835 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.835 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.835 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:42.839 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:43.074 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:43.074 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:43.075 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:43.075 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:43.076 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:43.076 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:43.076 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:43.077 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:43.078 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:43.093 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:43.094 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:43.094 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:43.094 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:43.094 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:43.095 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:43.095 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:43.095 [INFO - pool-44-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:43.099 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:43.393 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:43.394 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:43.395 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:43.395 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:43.395 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:43.395 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:43.395 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:43.395 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:43.398 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:43.473 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:43.473 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:43.474 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:43.474 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:43.474 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:43.474 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:43.474 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:43.474 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:43.477 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:43.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:43.709358 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:43.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:43.709682 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:44.093 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:44.094 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:44.095 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:44.095 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:44.095 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:44.096 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:44.096 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:44.097 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:44.099 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:44.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:44.636826 8567 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:44.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:44.637257 8567 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "618d653260f5431690fea318de34838f" member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } } peers { permanent_uuid: "c450723ed6be41008af218d230566e04" member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } } peers { permanent_uuid: "3f73f0cebe6f4dd89070e1842d3de8d9" member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } }
08:07:44.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:44.638860 8567 leader_election.cc:290] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 618d653260f5431690fea318de34838f (127.7.212.126:45529), c450723ed6be41008af218d230566e04 (127.7.212.125:41383)
08:07:44.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:44.646108 8484 leader_election.cc:336] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 618d653260f5431690fea318de34838f (127.7.212.126:45529): Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111)
08:07:44.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:44.646799 8484 leader_election.cc:336] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c450723ed6be41008af218d230566e04 (127.7.212.125:41383): Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:44.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:44.647138 8484 leader_election.cc:304] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3f73f0cebe6f4dd89070e1842d3de8d9; no voters: 618d653260f5431690fea318de34838f, c450723ed6be41008af218d230566e04
08:07:44.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:44.647704 8567 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:07:44.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:44.712679 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:44.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:44.712994 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:45.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:45.715981 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:45.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:45.716284 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:46.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:46.719290 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:46.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:46.719606 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:47.333 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:47.334 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:47.335 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:47.335 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:47.335 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:47.335 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:47.335 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:47.335 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:47.338 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:47.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:47.722402 8464 master_runner.cc:187] Error getting master registration for 127.7.212.126:45529: Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111),
08:07:47.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:47.722818 8464 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:07:47.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:47.723876 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:47.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:47.724289 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:47.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:47.944240 8574 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:47.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:47.944757 8574 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "618d653260f5431690fea318de34838f" member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } } peers { permanent_uuid: "c450723ed6be41008af218d230566e04" member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } } peers { permanent_uuid: "3f73f0cebe6f4dd89070e1842d3de8d9" member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } }
08:07:47.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:47.946569 8574 leader_election.cc:290] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 618d653260f5431690fea318de34838f (127.7.212.126:45529), c450723ed6be41008af218d230566e04 (127.7.212.125:41383)
08:07:47.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:47.948647 8484 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111) [suppressed 3 similar messages]
08:07:47.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:47.952734 8484 leader_election.cc:336] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 618d653260f5431690fea318de34838f (127.7.212.126:45529): Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111)
08:07:47.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:47.953603 8484 leader_election.cc:336] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c450723ed6be41008af218d230566e04 (127.7.212.125:41383): Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:47.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:47.953979 8484 leader_election.cc:304] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3f73f0cebe6f4dd89070e1842d3de8d9; no voters: 618d653260f5431690fea318de34838f, c450723ed6be41008af218d230566e04
08:07:47.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:47.954608 8574 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:07:48.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:48.727252 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:48.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:48.727581 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:49.273 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:49.273 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:49.274 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:49.274 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:49.274 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:49.274 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:49.274 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:49.274 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:49.278 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:49.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:49.538093 8578 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:49.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:49.538467 8578 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "618d653260f5431690fea318de34838f" member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } } peers { permanent_uuid: "c450723ed6be41008af218d230566e04" member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } } peers { permanent_uuid: "3f73f0cebe6f4dd89070e1842d3de8d9" member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } }
08:07:49.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:49.540069 8578 leader_election.cc:290] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 618d653260f5431690fea318de34838f (127.7.212.126:45529), c450723ed6be41008af218d230566e04 (127.7.212.125:41383)
08:07:49.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:49.545890 8484 leader_election.cc:336] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 618d653260f5431690fea318de34838f (127.7.212.126:45529): Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111)
08:07:49.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:49.546602 8484 leader_election.cc:336] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c450723ed6be41008af218d230566e04 (127.7.212.125:41383): Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:49.546959 8484 leader_election.cc:304] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3f73f0cebe6f4dd89070e1842d3de8d9; no voters: 618d653260f5431690fea318de34838f, c450723ed6be41008af218d230566e04
08:07:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:49.547677 8578 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:07:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:49.730644 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:49.730926 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:50.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:50.734632 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:50.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:50.734982 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:51.737942 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:51.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:51.738210 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:51.753 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:51.754 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:51.754 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:51.754 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:51.755 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:51.755 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:51.755 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:51.755 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:51.758 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:52.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:07:52 dist-test-slave-jt9t krb5kdc[8028](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856472, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:07:52.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:52.330050 8479 init.cc:295] Successfully reacquired a new kerberos TGT
08:07:52.594 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:52.595 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:52.595 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:52.595 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:52.596 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:52.596 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:52.596 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:52.596 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:52.599 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:52.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:52.726617 8464 master_runner.cc:187] Error getting master registration for 127.7.212.126:45529: Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111),
08:07:52.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:52.727079 8464 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:07:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:52.741110 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:52.741472 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:53.744333 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:53.744607 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:54.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:54.695225 7441 debug-util.cc:398] Leaking SignalData structure 0x7b08000984c0 after lost signal to thread 7376
08:07:54.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:54.695899 7441 debug-util.cc:398] Leaking SignalData structure 0x7b0800079080 after lost signal to thread 7444
08:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:54.747736 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:54.747993 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:55.733 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:55.734 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:55.734 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:55.734 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:55.735 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:55.735 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:55.735 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:55.735 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:55.740 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:55.751430 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:55.751694 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:56.754774 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:56.755039 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:56.893 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:56.894 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:56.894 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:56.894 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:56.894 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:56.895 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:56.895 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:56.895 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:56.897 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:57.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:57.107829 8593 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:57.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:57.108409 8593 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "618d653260f5431690fea318de34838f" member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } } peers { permanent_uuid: "c450723ed6be41008af218d230566e04" member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } } peers { permanent_uuid: "3f73f0cebe6f4dd89070e1842d3de8d9" member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } }
08:07:57.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:57.110394 8593 leader_election.cc:290] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 618d653260f5431690fea318de34838f (127.7.212.126:45529), c450723ed6be41008af218d230566e04 (127.7.212.125:41383)
08:07:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:57.113333 8484 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111) [suppressed 3 similar messages]
08:07:57.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:57.116676 8484 leader_election.cc:336] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 618d653260f5431690fea318de34838f (127.7.212.126:45529): Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111)
08:07:57.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:57.117370 8484 leader_election.cc:336] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c450723ed6be41008af218d230566e04 (127.7.212.125:41383): Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:57.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:57.117652 8484 leader_election.cc:304] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3f73f0cebe6f4dd89070e1842d3de8d9; no voters: 618d653260f5431690fea318de34838f, c450723ed6be41008af218d230566e04
08:07:57.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:57.118276 8593 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:07:57.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:57.730095 8464 master_runner.cc:187] Error getting master registration for 127.7.212.126:45529: Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111),
08:07:57.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:57.730461 8464 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:07:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:57.757943 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:57.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:57.758224 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:58.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:58.761125 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:58.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:58.761413 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:07:58.773 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:07:58.773 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:07:58.774 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:58.774 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:58.774 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:07:58.774 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:58.774 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:58.775 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:07:58.777 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:07:59.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:59.608320 8600 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:07:59.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:59.608779 8600 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "618d653260f5431690fea318de34838f" member_type: VOTER last_known_addr { host: "127.7.212.126" port: 45529 } } peers { permanent_uuid: "c450723ed6be41008af218d230566e04" member_type: VOTER last_known_addr { host: "127.7.212.125" port: 41383 } } peers { permanent_uuid: "3f73f0cebe6f4dd89070e1842d3de8d9" member_type: VOTER last_known_addr { host: "127.7.212.124" port: 35659 } }
08:07:59.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:59.610570 8600 leader_election.cc:290] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 618d653260f5431690fea318de34838f (127.7.212.126:45529), c450723ed6be41008af218d230566e04 (127.7.212.125:41383)
08:07:59.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:59.616348 8484 leader_election.cc:336] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 618d653260f5431690fea318de34838f (127.7.212.126:45529): Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111)
08:07:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:59.616945 8484 leader_election.cc:336] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c450723ed6be41008af218d230566e04 (127.7.212.125:41383): Network error: Client connection negotiation failed: client connection to 127.7.212.125:41383: connect: Connection refused (error 111)
08:07:59.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:59.617261 8484 leader_election.cc:304] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3f73f0cebe6f4dd89070e1842d3de8d9; no voters: 618d653260f5431690fea318de34838f, c450723ed6be41008af218d230566e04
08:07:59.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:07:59.617961 8600 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:07:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:59.764727 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:07:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:07:59.765107 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:00.113 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:08:00.115 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:08:00.116 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:00.116 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:00.116 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:00.117 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:00.117 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:00.117 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:00.119 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:08:00.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:00.769421 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:00.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:00.769893 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:01.773265 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:01.773610 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:02.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 08:08:02 dist-test-slave-jt9t krb5kdc[8028](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781856482, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.212.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:02.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:08:02.332902 8479 init.cc:295] Successfully reacquired a new kerberos TGT
08:08:02.434 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:08:02.435 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:02.435 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:02.436 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:02.436 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:08:02.437 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:02.437 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:02.437 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:02.441 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:08:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:08:02.734191 8464 master_runner.cc:187] Error getting master registration for 127.7.212.126:45529: Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111),
08:08:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:08:02.734611 8464 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:08:02.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:02.776888 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:02.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:02.777222 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:03.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:03.780408 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:03.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:03.780740 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:04.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:04.783725 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:04.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:04.784055 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:05.787293 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:05.787636 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:06.514 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:08:06.515 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:08:06.516 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:06.516 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:06.516 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:06.516 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:06.516 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:06.516 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:06.519 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:08:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:06.790694 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:06.791016 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:07.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:08:07.737571 8464 master_runner.cc:187] Error getting master registration for 127.7.212.126:45529: Network error: Client connection negotiation failed: client connection to 127.7.212.126:45529: connect: Connection refused (error 111),
08:08:07.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:08:07.738047 8464 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:08:07.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:07.794215 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:07.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:07.794515 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:08.353 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:08:08.354 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:08:08.354 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:08.355 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:08.355 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:08.355 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:08.355 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:08.355 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:08.358 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:08:08.573 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:08:08.574 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:08:08.575 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:08.575 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:08.575 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:08.575 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:08.575 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:08.576 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:08.579 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:08:08.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:08.797542 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:08.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:08.797878 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:09.800916 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:09.801257 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:10.805090 8563 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3f73f0cebe6f4dd89070e1842d3de8d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:10.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 08:08:10.805542 8563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:11.253 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.125:41383(127.7.212.125:41383)] connecting to peer
08:08:11.254 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.7.212.126:45529(127.7.212.126:45529)] connecting to peer
08:08:11.254 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:11.254 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.7.212.125:41383(127.7.212.125:41383)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:11.254 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:11.254 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.125:41383: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383
08:08:11.255 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.7.212.126:45529(127.7.212.126:45529)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:11.255 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.7.212.126:45529: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
08:08:11.258 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659; will retry
08:08:11.259 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28746), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [4ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [8ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [73ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [74ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [97ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [97ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [100ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [102ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [102ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [124ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [128ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [134ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [134ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [145ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [151ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [152ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [153ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [184ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [184ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [187ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [187ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [223ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [223ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [226ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [227ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [261ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [263ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [263ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [264ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [267ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [267ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [321ms] refreshing cache from master, [321ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [323ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [323ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [326ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [327ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [561ms] refreshing cache from master, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [564ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [565ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [566ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [580ms] refreshing cache from master, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [582ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [583ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [583ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [587ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [587ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [881ms] refreshing cache from master, [881ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [883ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [883ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [883ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [886ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [960ms] refreshing cache from master, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [962ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [962ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [965ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [965ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [1581ms] refreshing cache from master, [1581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [1582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [1583ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [1583ms] too many traces: truncated at 100 traces, deferred=Deferred@55696701(state=PENDING, result=null, callback=(continuation of Deferred@565450773 after retry RPC after error@1718726987) -> (continuation of Deferred@1580276295 after retry RPC after error@1392807049) -> (continuation of Deferred@1391542860 after retry RPC after error@962677424) -> (continuation of Deferred@1876204793 after retry RPC after error@415442254) -> (continuation of Deferred@1081669432 after retry RPC after error@1946095949) -> (continuation of Deferred@948402335 after retry RPC after error@972014484) -> (continuation of Deferred@1042883238 after retry RPC after error@903297367) -> (continuation of Deferred@915930354 after retry RPC after error@1906535127) -> (continuation of Deferred@107502167 after retry RPC after error@184141388) -> (continuation of Deferred@50383757 after retry RPC after error@1763699944) -> (continuation of Deferred@769763710 after retry RPC after error@1768870184) -> (continuation of Deferred@1815556907 after retry RPC after error@2052717289) -> (continuation of Deferred@1515394693 after retry RPC after error@1160473462) -> (continuation of Deferred@282113683 after retry RPC after error@627808466) -> (continuation of Deferred@1733187012 after retry RPC after error@2040590976) -> (continuation of Deferred@1825679113 after retry RPC after error@361045833) -> (continuation of Deferred@7374920 after retry RPC after error@2126138932) -> (continuation of Deferred@523392773 after retry RPC after error@577793277) -> (continuation of Deferred@650543667 after retry RPC after error@169976625) -> (continuation of Deferred@128341775 after retry RPC after error@647021698) -> (continuation of Deferred@877989241 after retry RPC after error@862591209) -> (continuation of Deferred@104935332 after retry RPC after error@643125050) -> (continuation of Deferred@989306956 after retry RPC after error@1496930987) -> (continuation of Deferred@1443521063 after retry RPC after error@1512758083) -> (continuation of Deferred@1566107018 after retry RPC after error@1311182984), errback=(continuation of Deferred@565450773 after retry RPC after error@1718726987) -> (continuation of Deferred@1580276295 after retry RPC after error@1392807049) -> (continuation of Deferred@1391542860 after retry RPC after error@962677424) -> (continuation of Deferred@1876204793 after retry RPC after error@415442254) -> (continuation of Deferred@1081669432 after retry RPC after error@1946095949) -> (continuation of Deferred@948402335 after retry RPC after error@972014484) -> (continuation of Deferred@1042883238 after retry RPC after error@903297367) -> (continuation of Deferred@915930354 after retry RPC after error@1906535127) -> (continuation of Deferred@107502167 after retry RPC after error@184141388) -> (continuation of Deferred@50383757 after retry RPC after error@1763699944) -> (continuation of Deferred@769763710 after retry RPC after error@1768870184) -> (continuation of Deferred@1815556907 after retry RPC after error@2052717289) -> (continuation of Deferred@1515394693 after retry RPC after error@1160473462) -> (continuation of Deferred@282113683 after retry RPC after error@627808466) -> (continuation of Deferred@1733187012 after retry RPC after error@2040590976) -> (continuation of Deferred@1825679113 after retry RPC after error@361045833) -> (continuation of Deferred@7374920 after retry RPC after error@2126138932) -> (continuation of Deferred@523392773 after retry RPC after error@577793277) -> (continuation of Deferred@650543667 after retry RPC after error@169976625) -> (continuation of Deferred@128341775 after retry RPC after error@647021698) -> (continuation of Deferred@877989241 after retry RPC after error@862591209) -> (continuation of Deferred@104935332 after retry RPC after error@643125050) -> (continuation of Deferred@989306956 after retry RPC after error@1496930987) -> (continuation of Deferred@1443521063 after retry RPC after error@1512758083) -> (continuation of Deferred@1566107018 after retry RPC after error@1311182984)))
08:08:11.262 [DEBUG - main] (Connection.java:575) [peer master-127.7.212.125:41383(127.7.212.125:41383)] disconnecting while in state TERMINATED
08:08:11.262 [DEBUG - main] (Connection.java:575) [peer master-127.7.212.124:35659(127.7.212.124:35659)] disconnecting while in state READY
08:08:11.262 [DEBUG - main] (Connection.java:575) [peer master-127.7.212.126:45529(127.7.212.126:45529)] disconnecting while in state TERMINATED
08:08:11.263 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:08:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:08:11.265427 8017 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu with pid 8464
08:08:11.265 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.7.212.124:35659(127.7.212.124:35659)] handling channelInactive
08:08:11.266 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.7.212.124:35659(127.7.212.124:35659)] cleaning up while in state READY due to: connection closed
08:08:12.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T08:08:12Z chronyd exiting
08:08:13.510 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:08:13.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:08:13.513082 7290 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu with pid 7584
08:08:13.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:08:13.539898 7290 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu with pid 7727
08:08:13.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:08:13.565510 7290 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu with pid 7875
08:08:13.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:08:13.592115 7290 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu with pid 7302
08:08:13.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:08:13.626757 7290 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu with pid 7372
08:08:13.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 08:08:13.661684 7290 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskfpJR7l/build/tsan/bin/kudu with pid 7447
08:08:14.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T08:08:14Z chronyd exiting
08:08:15.862 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28746), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [4ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [8ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [73ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [74ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [97ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [97ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [100ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [102ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [102ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [124ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [128ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [134ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [134ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [145ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [151ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [152ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [153ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [184ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [184ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [187ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [187ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [223ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [223ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [226ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [227ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [261ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [263ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [263ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [264ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [267ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [267ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [321ms] refreshing cache from master, [321ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [323ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [323ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [326ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [327ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [561ms] refreshing cache from master, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [564ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [565ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [566ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [580ms] refreshing cache from master, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [582ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [583ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [583ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [587ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [587ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [881ms] refreshing cache from master, [881ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [883ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [883ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [883ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [886ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [960ms] refreshing cache from master, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [962ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [962ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [965ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [965ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [1581ms] refreshing cache from master, [1581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [1582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [1583ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [1583ms] too many traces: truncated at 100 traces, deferred=Deferred@55696701(state=PENDING, result=null, callback=(continuation of Deferred@565450773 after retry RPC after error@1718726987) -> (continuation of Deferred@1580276295 after retry RPC after error@1392807049) -> (continuation of Deferred@1391542860 after retry RPC after error@962677424) -> (continuation of Deferred@1876204793 after retry RPC after error@415442254) -> (continuation of Deferred@1081669432 after retry RPC after error@1946095949) -> (continuation of Deferred@948402335 after retry RPC after error@972014484) -> (continuation of Deferred@1042883238 after retry RPC after error@903297367) -> (continuation of Deferred@915930354 after retry RPC after error@1906535127) -> (continuation of Deferred@107502167 after retry RPC after error@184141388) -> (continuation of Deferred@50383757 after retry RPC after error@1763699944) -> (continuation of Deferred@769763710 after retry RPC after error@1768870184) -> (continuation of Deferred@1815556907 after retry RPC after error@2052717289) -> (continuation of Deferred@1515394693 after retry RPC after error@1160473462) -> (continuation of Deferred@282113683 after retry RPC after error@627808466) -> (continuation of Deferred@1733187012 after retry RPC after error@2040590976) -> (continuation of Deferred@1825679113 after retry RPC after error@361045833) -> (continuation of Deferred@7374920 after retry RPC after error@2126138932) -> (continuation of Deferred@523392773 after retry RPC after error@577793277) -> (continuation of Deferred@650543667 after retry RPC after error@169976625) -> (continuation of Deferred@128341775 after retry RPC after error@647021698) -> (continuation of Deferred@877989241 after retry RPC after error@862591209) -> (continuation of Deferred@104935332 after retry RPC after error@643125050) -> (continuation of Deferred@989306956 after retry RPC after error@1496930987) -> (continuation of Deferred@1443521063 after retry RPC after error@1512758083) -> (continuation of Deferred@1566107018 after retry RPC after error@1311182984), errback=(continuation of Deferred@565450773 after retry RPC after error@1718726987) -> (continuation of Deferred@1580276295 after retry RPC after error@1392807049) -> (continuation of Deferred@1391542860 after retry RPC after error@962677424) -> (continuation of Deferred@1876204793 after retry RPC after error@415442254) -> (continuation of Deferred@1081669432 after retry RPC after error@1946095949) -> (continuation of Deferred@948402335 after retry RPC after error@972014484) -> (continuation of Deferred@1042883238 after retry RPC after error@903297367) -> (continuation of Deferred@915930354 after retry RPC after error@1906535127) -> (continuation of Deferred@107502167 after retry RPC after error@184141388) -> (continuation of Deferred@50383757 after retry RPC after error@1763699944) -> (continuation of Deferred@769763710 after retry RPC after error@1768870184) -> (continuation of Deferred@1815556907 after retry RPC after error@2052717289) -> (continuation of Deferred@1515394693 after retry RPC after error@1160473462) -> (continuation of Deferred@282113683 after retry RPC after error@627808466) -> (continuation of Deferred@1733187012 after retry RPC after error@2040590976) -> (continuation of Deferred@1825679113 after retry RPC after error@361045833) -> (continuation of Deferred@7374920 after retry RPC after error@2126138932) -> (continuation of Deferred@523392773 after retry RPC after error@577793277) -> (continuation of Deferred@650543667 after retry RPC after error@169976625) -> (continuation of Deferred@128341775 after retry RPC after error@647021698) -> (continuation of Deferred@877989241 after retry RPC after error@862591209) -> (continuation of Deferred@104935332 after retry RPC after error@643125050) -> (continuation of Deferred@989306956 after retry RPC after error@1496930987) -> (continuation of Deferred@1443521063 after retry RPC after error@1512758083) -> (continuation of Deferred@1566107018 after retry RPC after error@1311182984)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.212.126:45529
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28746), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [4ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [8ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [73ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [74ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [97ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [97ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [100ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [102ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [102ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [124ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [128ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [134ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [134ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [145ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [151ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [152ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [153ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [184ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [184ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [187ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [187ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [223ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [223ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [226ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [227ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [261ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [263ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [263ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [264ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [267ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [267ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [321ms] refreshing cache from master, [321ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [323ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [323ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [326ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [327ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [561ms] refreshing cache from master, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [564ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [565ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [566ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [580ms] refreshing cache from master, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [582ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [583ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [583ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [587ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [587ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [881ms] refreshing cache from master, [881ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [883ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [883ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [883ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [886ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [960ms] refreshing cache from master, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [962ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [962ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [965ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [965ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [1581ms] refreshing cache from master, [1581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [1582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [1583ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [1583ms] too many traces: truncated at 100 traces, deferred=Deferred@55696701(state=PENDING, result=null, callback=(continuation of Deferred@565450773 after retry RPC after error@1718726987) -> (continuation of Deferred@1580276295 after retry RPC after error@1392807049) -> (continuation of Deferred@1391542860 after retry RPC after error@962677424) -> (continuation of Deferred@1876204793 after retry RPC after error@415442254) -> (continuation of Deferred@1081669432 after retry RPC after error@1946095949) -> (continuation of Deferred@948402335 after retry RPC after error@972014484) -> (continuation of Deferred@1042883238 after retry RPC after error@903297367) -> (continuation of Deferred@915930354 after retry RPC after error@1906535127) -> (continuation of Deferred@107502167 after retry RPC after error@184141388) -> (continuation of Deferred@50383757 after retry RPC after error@1763699944) -> (continuation of Deferred@769763710 after retry RPC after error@1768870184) -> (continuation of Deferred@1815556907 after retry RPC after error@2052717289) -> (continuation of Deferred@1515394693 after retry RPC after error@1160473462) -> (continuation of Deferred@282113683 after retry RPC after error@627808466) -> (continuation of Deferred@1733187012 after retry RPC after error@2040590976) -> (continuation of Deferred@1825679113 after retry RPC after error@361045833) -> (continuation of Deferred@7374920 after retry RPC after error@2126138932) -> (continuation of Deferred@523392773 after retry RPC after error@577793277) -> (continuation of Deferred@650543667 after retry RPC after error@169976625) -> (continuation of Deferred@128341775 after retry RPC after error@647021698) -> (continuation of Deferred@877989241 after retry RPC after error@862591209) -> (continuation of Deferred@104935332 after retry RPC after error@643125050) -> (continuation of Deferred@989306956 after retry RPC after error@1496930987) -> (continuation of Deferred@1443521063 after retry RPC after error@1512758083) -> (continuation of Deferred@1566107018 after retry RPC after error@1311182984), errback=(continuation of Deferred@565450773 after retry RPC after error@1718726987) -> (continuation of Deferred@1580276295 after retry RPC after error@1392807049) -> (continuation of Deferred@1391542860 after retry RPC after error@962677424) -> (continuation of Deferred@1876204793 after retry RPC after error@415442254) -> (continuation of Deferred@1081669432 after retry RPC after error@1946095949) -> (continuation of Deferred@948402335 after retry RPC after error@972014484) -> (continuation of Deferred@1042883238 after retry RPC after error@903297367) -> (continuation of Deferred@915930354 after retry RPC after error@1906535127) -> (continuation of Deferred@107502167 after retry RPC after error@184141388) -> (continuation of Deferred@50383757 after retry RPC after error@1763699944) -> (continuation of Deferred@769763710 after retry RPC after error@1768870184) -> (continuation of Deferred@1815556907 after retry RPC after error@2052717289) -> (continuation of Deferred@1515394693 after retry RPC after error@1160473462) -> (continuation of Deferred@282113683 after retry RPC after error@627808466) -> (continuation of Deferred@1733187012 after retry RPC after error@2040590976) -> (continuation of Deferred@1825679113 after retry RPC after error@361045833) -> (continuation of Deferred@7374920 after retry RPC after error@2126138932) -> (continuation of Deferred@523392773 after retry RPC after error@577793277) -> (continuation of Deferred@650543667 after retry RPC after error@169976625) -> (continuation of Deferred@128341775 after retry RPC after error@647021698) -> (continuation of Deferred@877989241 after retry RPC after error@862591209) -> (continuation of Deferred@104935332 after retry RPC after error@643125050) -> (continuation of Deferred@989306956 after retry RPC after error@1496930987) -> (continuation of Deferred@1443521063 after retry RPC after error@1512758083) -> (continuation of Deferred@1566107018 after retry RPC after error@1311182984)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.212.126:45529
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
08:08:16.437 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: giving up after 1 attempts
08:08:16.437 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28746), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [4ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [8ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [73ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [74ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [97ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [97ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [100ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [102ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [102ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [124ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [128ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [134ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [134ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [145ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [151ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [152ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [153ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [184ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [184ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [187ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [187ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [223ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [223ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [226ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [227ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [261ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [263ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [263ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [264ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [267ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [267ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [321ms] refreshing cache from master, [321ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [323ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [323ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [326ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [327ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [561ms] refreshing cache from master, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [564ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [565ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [566ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [580ms] refreshing cache from master, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [582ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [583ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [583ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [587ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [587ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [881ms] refreshing cache from master, [881ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [883ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [883ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [883ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [886ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [960ms] refreshing cache from master, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [962ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [962ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [965ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [965ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [1581ms] refreshing cache from master, [1581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [1582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [1583ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [1583ms] too many traces: truncated at 100 traces, deferred=Deferred@55696701(state=PENDING, result=null, callback=(continuation of Deferred@565450773 after retry RPC after error@1718726987) -> (continuation of Deferred@1580276295 after retry RPC after error@1392807049) -> (continuation of Deferred@1391542860 after retry RPC after error@962677424) -> (continuation of Deferred@1876204793 after retry RPC after error@415442254) -> (continuation of Deferred@1081669432 after retry RPC after error@1946095949) -> (continuation of Deferred@948402335 after retry RPC after error@972014484) -> (continuation of Deferred@1042883238 after retry RPC after error@903297367) -> (continuation of Deferred@915930354 after retry RPC after error@1906535127) -> (continuation of Deferred@107502167 after retry RPC after error@184141388) -> (continuation of Deferred@50383757 after retry RPC after error@1763699944) -> (continuation of Deferred@769763710 after retry RPC after error@1768870184) -> (continuation of Deferred@1815556907 after retry RPC after error@2052717289) -> (continuation of Deferred@1515394693 after retry RPC after error@1160473462) -> (continuation of Deferred@282113683 after retry RPC after error@627808466) -> (continuation of Deferred@1733187012 after retry RPC after error@2040590976) -> (continuation of Deferred@1825679113 after retry RPC after error@361045833) -> (continuation of Deferred@7374920 after retry RPC after error@2126138932) -> (continuation of Deferred@523392773 after retry RPC after error@577793277) -> (continuation of Deferred@650543667 after retry RPC after error@169976625) -> (continuation of Deferred@128341775 after retry RPC after error@647021698) -> (continuation of Deferred@877989241 after retry RPC after error@862591209) -> (continuation of Deferred@104935332 after retry RPC after error@643125050) -> (continuation of Deferred@989306956 after retry RPC after error@1496930987) -> (continuation of Deferred@1443521063 after retry RPC after error@1512758083) -> (continuation of Deferred@1566107018 after retry RPC after error@1311182984), errback=(continuation of Deferred@565450773 after retry RPC after error@1718726987) -> (continuation of Deferred@1580276295 after retry RPC after error@1392807049) -> (continuation of Deferred@1391542860 after retry RPC after error@962677424) -> (continuation of Deferred@1876204793 after retry RPC after error@415442254) -> (continuation of Deferred@1081669432 after retry RPC after error@1946095949) -> (continuation of Deferred@948402335 after retry RPC after error@972014484) -> (continuation of Deferred@1042883238 after retry RPC after error@903297367) -> (continuation of Deferred@915930354 after retry RPC after error@1906535127) -> (continuation of Deferred@107502167 after retry RPC after error@184141388) -> (continuation of Deferred@50383757 after retry RPC after error@1763699944) -> (continuation of Deferred@769763710 after retry RPC after error@1768870184) -> (continuation of Deferred@1815556907 after retry RPC after error@2052717289) -> (continuation of Deferred@1515394693 after retry RPC after error@1160473462) -> (continuation of Deferred@282113683 after retry RPC after error@627808466) -> (continuation of Deferred@1733187012 after retry RPC after error@2040590976) -> (continuation of Deferred@1825679113 after retry RPC after error@361045833) -> (continuation of Deferred@7374920 after retry RPC after error@2126138932) -> (continuation of Deferred@523392773 after retry RPC after error@577793277) -> (continuation of Deferred@650543667 after retry RPC after error@169976625) -> (continuation of Deferred@128341775 after retry RPC after error@647021698) -> (continuation of Deferred@877989241 after retry RPC after error@862591209) -> (continuation of Deferred@104935332 after retry RPC after error@643125050) -> (continuation of Deferred@989306956 after retry RPC after error@1496930987) -> (continuation of Deferred@1443521063 after retry RPC after error@1512758083) -> (continuation of Deferred@1566107018 after retry RPC after error@1311182984)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.212.126:45529
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=28746), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [4ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [5ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [8ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [73ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [74ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [97ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [97ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [100ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [102ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [102ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [124ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [128ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [134ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [134ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [145ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [151ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [152ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [153ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [184ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [184ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [187ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [187ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [223ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [223ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [226ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [227ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [261ms] refreshing cache from master, [261ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [263ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [263ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [264ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [267ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [267ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [321ms] refreshing cache from master, [321ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [322ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [323ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [323ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [326ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [327ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [561ms] refreshing cache from master, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [564ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [565ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [566ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [580ms] refreshing cache from master, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [582ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [583ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [583ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [587ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [587ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [881ms] refreshing cache from master, [881ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [883ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [883ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [883ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [886ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [960ms] refreshing cache from master, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.124:35659, [962ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [962ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.126:45529: Network error: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [965ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.124:35659: OK, [965ms] delaying RPC due to: Service unavailable: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529, [1581ms] refreshing cache from master, [1581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.125:41383, [1582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.212.126:45529, [1583ms] Sub RPC ConnectToMaster: received response from server master-127.7.212.125:41383: Network error: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383, [1583ms] too many traces: truncated at 100 traces, deferred=Deferred@55696701(state=PENDING, result=null, callback=(continuation of Deferred@565450773 after retry RPC after error@1718726987) -> (continuation of Deferred@1580276295 after retry RPC after error@1392807049) -> (continuation of Deferred@1391542860 after retry RPC after error@962677424) -> (continuation of Deferred@1876204793 after retry RPC after error@415442254) -> (continuation of Deferred@1081669432 after retry RPC after error@1946095949) -> (continuation of Deferred@948402335 after retry RPC after error@972014484) -> (continuation of Deferred@1042883238 after retry RPC after error@903297367) -> (continuation of Deferred@915930354 after retry RPC after error@1906535127) -> (continuation of Deferred@107502167 after retry RPC after error@184141388) -> (continuation of Deferred@50383757 after retry RPC after error@1763699944) -> (continuation of Deferred@769763710 after retry RPC after error@1768870184) -> (continuation of Deferred@1815556907 after retry RPC after error@2052717289) -> (continuation of Deferred@1515394693 after retry RPC after error@1160473462) -> (continuation of Deferred@282113683 after retry RPC after error@627808466) -> (continuation of Deferred@1733187012 after retry RPC after error@2040590976) -> (continuation of Deferred@1825679113 after retry RPC after error@361045833) -> (continuation of Deferred@7374920 after retry RPC after error@2126138932) -> (continuation of Deferred@523392773 after retry RPC after error@577793277) -> (continuation of Deferred@650543667 after retry RPC after error@169976625) -> (continuation of Deferred@128341775 after retry RPC after error@647021698) -> (continuation of Deferred@877989241 after retry RPC after error@862591209) -> (continuation of Deferred@104935332 after retry RPC after error@643125050) -> (continuation of Deferred@989306956 after retry RPC after error@1496930987) -> (continuation of Deferred@1443521063 after retry RPC after error@1512758083) -> (continuation of Deferred@1566107018 after retry RPC after error@1311182984), errback=(continuation of Deferred@565450773 after retry RPC after error@1718726987) -> (continuation of Deferred@1580276295 after retry RPC after error@1392807049) -> (continuation of Deferred@1391542860 after retry RPC after error@962677424) -> (continuation of Deferred@1876204793 after retry RPC after error@415442254) -> (continuation of Deferred@1081669432 after retry RPC after error@1946095949) -> (continuation of Deferred@948402335 after retry RPC after error@972014484) -> (continuation of Deferred@1042883238 after retry RPC after error@903297367) -> (continuation of Deferred@915930354 after retry RPC after error@1906535127) -> (continuation of Deferred@107502167 after retry RPC after error@184141388) -> (continuation of Deferred@50383757 after retry RPC after error@1763699944) -> (continuation of Deferred@769763710 after retry RPC after error@1768870184) -> (continuation of Deferred@1815556907 after retry RPC after error@2052717289) -> (continuation of Deferred@1515394693 after retry RPC after error@1160473462) -> (continuation of Deferred@282113683 after retry RPC after error@627808466) -> (continuation of Deferred@1733187012 after retry RPC after error@2040590976) -> (continuation of Deferred@1825679113 after retry RPC after error@361045833) -> (continuation of Deferred@7374920 after retry RPC after error@2126138932) -> (continuation of Deferred@523392773 after retry RPC after error@577793277) -> (continuation of Deferred@650543667 after retry RPC after error@169976625) -> (continuation of Deferred@128341775 after retry RPC after error@647021698) -> (continuation of Deferred@877989241 after retry RPC after error@862591209) -> (continuation of Deferred@104935332 after retry RPC after error@643125050) -> (continuation of Deferred@989306956 after retry RPC after error@1496930987) -> (continuation of Deferred@1443521063 after retry RPC after error@1512758083) -> (continuation of Deferred@1566107018 after retry RPC after error@1311182984)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.212.125:41383,127.7.212.126:45529,127.7.212.124:35659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.125:41383(127.7.212.125:41383): Connection refused: /127.7.212.125:41383,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.212.126:45529(127.7.212.126:45529): Connection refused: /127.7.212.126:45529
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.212.126:45529
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)