08:07:58.640 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:07:58.640 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskbTrDWn/test-tmp
08:07:58.640 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:07:58.640 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskbTrDWn/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:07:58.643 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:07:58.992 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:07:58.992 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:07:58.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:07:58.996598 29409 mini_oidc.cc:150] Starting JWKS server
08:07:59.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:07:59.000329 29409 webserver.cc:533] Webserver started at http://127.0.0.1:38709/ using document root <none> and password file <none>
08:07:59.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:07:59.000811 29409 mini_oidc.cc:186] Starting OIDC Discovery server
08:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:07:59.002621 29409 webserver.cc:533] Webserver started at http://127.0.0.1:46781/ using document root <none> and password file <none>
08:07:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-18T08:07:59Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:07:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-18T08:07:59Z Disabled control of system clock
08:07:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:07:59.070492 29409 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/wal
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/data
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/logs
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/data/info.pb
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.184.126:39579
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.184.126
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.184.84:36815
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.184.126:39579,127.28.184.125:44265,127.28.184.124:42655
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38709/jwks.json with env {}
08:07:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:07:59.376335 29421 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:07:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:07:59.376982 29421 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:07:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:07:59.377440 29421 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:07:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:07:59.411278 29421 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:07:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:07:59.411581 29421 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:07:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:07:59.411779 29421 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:07:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:07:59.412102 29421 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:07:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:07:59.412391 29421 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:07:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:07:59.412591 29421 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38709/jwks.json
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:07:59.450594 29421 master_runner.cc:387] Master server non-default flags:
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.184.84:36815
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/data
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/wal
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.184.126:39579,127.28.184.125:44265,127.28.184.124:42655
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.184.126:39579
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38709/jwks.json
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/data/info.pb
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.184.126
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/logs
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 07:43:20 UTC on e7f111948823
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12451
08:07:59.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:07:59.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:07:59.451925 29421 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:07:59.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:07:59.453612 29421 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:07:59.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:07:59.466068 29427 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:01.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:00.870620 29426 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 29421
08:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.282905 29426 kernel_stack_watchdog.cc:198] Thread 29421 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
08:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:08:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:07:59.467535 29428 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.284554 29421 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.818s user 0.000s sys 0.002s
08:08:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.284906 29421 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.819s user 0.000s sys 0.002s
08:08:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.285086 29429 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1816 milliseconds
08:08:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.288249 29421 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.288932 29430 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.291002 29421 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:01.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.293972 29421 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:01.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.295581 29421 hybrid_clock.cc:648] HybridClock initialized: now 1781770081295534 us; error 38 us; skew 500 ppm
08:08:01.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.296609 29421 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.303778 29421 webserver.cc:533] Webserver started at http://127.28.184.126:39989/ using document root <none> and password file <none>
08:08:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.304903 29421 fs_manager.cc:362] Metadata directory not provided
08:08:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.305187 29421 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.305711 29421 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:08:01.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.310510 29421 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/data/instance:
08:08:01.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f538a1f9c19b473a9c7aff0155f0fc71"
08:08:01.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:01 on dist-test-slave-d25z"
08:08:01.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.311823 29421 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/wal/instance:
08:08:01.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f538a1f9c19b473a9c7aff0155f0fc71"
08:08:01.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:01 on dist-test-slave-d25z"
08:08:01.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.320125 29421 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.005s
08:08:01.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.326515 29437 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:01.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.327674 29421 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
08:08:01.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.328058 29421 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/data,/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/wal
08:08:01.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f538a1f9c19b473a9c7aff0155f0fc71"
08:08:01.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:01 on dist-test-slave-d25z"
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.328460 29421 fs_report.cc:389] FS layout report
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/wal
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/wal
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/data/data
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:01.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.406461 29421 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:01.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.408257 29421 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:01.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.408823 29421 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.492798 29421 rpc_server.cc:307] RPC server started. Bound to: 127.28.184.126:39579
08:08:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.492901 29488 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.184.126:39579 every 8 connection(s)
08:08:01.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.495828 29421 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/data/info.pb
08:08:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.498476 29409 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu as pid 29421
08:08:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.499066 29409 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-0/wal/instance
08:08:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.504089 29489 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.504846 29409 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/wal
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/data
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/logs
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/data/info.pb
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.184.125:44265
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.184.125
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.184.84:36815
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.184.126:39579,127.28.184.125:44265,127.28.184.124:42655
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38709/jwks.json with env {}
08:08:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.518816 29489 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } has no permanent_uuid. Determining permanent_uuid...
08:08:01.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.553696 29489 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } has no permanent_uuid. Determining permanent_uuid...
08:08:01.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.556715 29441 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.184.125:44265: connect: Connection refused (error 111)
08:08:01.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.562179 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.125:44265: Network error: Client connection negotiation failed: client connection to 127.28.184.125:44265: connect: Connection refused (error 111)
08:08:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.611749 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } attempt: 1
08:08:01.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.617511 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.125:44265: Network error: Client connection negotiation failed: client connection to 127.28.184.125:44265: connect: Connection refused (error 111)
08:08:01.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.685981 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } attempt: 2
08:08:01.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.691623 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.125:44265: Network error: Client connection negotiation failed: client connection to 127.28.184.125:44265: connect: Connection refused (error 111)
08:08:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.783109 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } attempt: 3
08:08:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.787675 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.125:44265: Network error: Client connection negotiation failed: client connection to 127.28.184.125:44265: connect: Connection refused (error 111)
08:08:01.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.858870 29491 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:01.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.859494 29491 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:01.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.859982 29491 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:01.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.895675 29491 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:01.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.895995 29491 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:01.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.896201 29491 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:01.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.896412 29491 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:01.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.896633 29491 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:01.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.896795 29491 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38709/jwks.json
08:08:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.931166 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } attempt: 4
08:08:01.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.935886 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.125:44265: Network error: Client connection negotiation failed: client connection to 127.28.184.125:44265: connect: Connection refused (error 111)
08:08:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.937127 29491 master_runner.cc:387] Master server non-default flags:
08:08:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.184.84:36815
08:08:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/data
08:08:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/wal
08:08:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.184.126:39579,127.28.184.125:44265,127.28.184.124:42655
08:08:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.184.125:44265
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38709/jwks.json
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/data/info.pb
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.184.125
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/logs
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 07:43:20 UTC on e7f111948823
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12451
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:01.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.938766 29491 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:01.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:01.940448 29491 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:01.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.952144 29500 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:02.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:02.235333 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } attempt: 5
08:08:02.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:02.244565 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.125:44265: Network error: Client connection negotiation failed: client connection to 127.28.184.125:44265: connect: Connection refused (error 111)
08:08:02.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:02.792187 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } attempt: 6
08:08:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:02.816063 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.125:44265: Network error: Client connection negotiation failed: client connection to 127.28.184.125:44265: connect: Connection refused (error 111)
08:08:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:01.954141 29501 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:03.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.262774 29502 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1301 milliseconds
08:08:03.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.264983 29503 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:03.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.267521 29491 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.314s user 0.001s sys 0.010s
08:08:03.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.267807 29491 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.315s user 0.001s sys 0.010s
08:08:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.267989 29491 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:03.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.269292 29491 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.274286 29491 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.275688 29491 hybrid_clock.cc:648] HybridClock initialized: now 1781770083275628 us; error 73 us; skew 500 ppm
08:08:03.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.276677 29491 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:03.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.285960 29491 webserver.cc:533] Webserver started at http://127.28.184.125:39417/ using document root <none> and password file <none>
08:08:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.286991 29491 fs_manager.cc:362] Metadata directory not provided
08:08:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.287263 29491 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.287791 29491 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:08:03.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.292655 29491 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/data/instance:
08:08:03.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29d7f53fb1244eed8714a8801297d276"
08:08:03.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:03 on dist-test-slave-d25z"
08:08:03.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.293757 29491 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/wal/instance:
08:08:03.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29d7f53fb1244eed8714a8801297d276"
08:08:03.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:03 on dist-test-slave-d25z"
08:08:03.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.302505 29491 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
08:08:03.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.309520 29512 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:03.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.310829 29491 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
08:08:03.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.311127 29491 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/data,/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/wal
08:08:03.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29d7f53fb1244eed8714a8801297d276"
08:08:03.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:03 on dist-test-slave-d25z"
08:08:03.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.311470 29491 fs_report.cc:389] FS layout report
08:08:03.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:03.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/wal
08:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/wal
08:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/data/data
08:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:03.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.390976 29491 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.392750 29491 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.393198 29491 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:03.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.476867 29491 rpc_server.cc:307] RPC server started. Bound to: 127.28.184.125:44265
08:08:03.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.476989 29563 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.184.125:44265 every 8 connection(s)
08:08:03.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.479804 29491 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/data/info.pb
08:08:03.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.481237 29409 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu as pid 29491
08:08:03.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.481647 29409 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-1/wal/instance
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.484853 29409 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/wal
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/data
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/logs
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/data/info.pb
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:03.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.184.124:42655
08:08:03.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.184.124
08:08:03.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:03.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.184.84:36815
08:08:03.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:03.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:03.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:03.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:03.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.184.126:39579,127.28.184.125:44265,127.28.184.124:42655
08:08:03.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:03.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38709/jwks.json with env {}
08:08:03.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.487828 29564 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:08:03.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.504067 29564 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } has no permanent_uuid. Determining permanent_uuid...
08:08:03.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.529598 29564 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } has no permanent_uuid. Determining permanent_uuid...
08:08:03.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.544855 29564 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } has no permanent_uuid. Determining permanent_uuid...
08:08:03.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.547698 29513 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:03.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.552268 29564 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.601895 29564 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 1
08:08:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.607281 29564 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:03.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.675712 29564 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 2
08:08:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.680569 29564 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:03.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.771963 29564 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 3
08:08:03.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.776993 29564 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:03.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.835299 29566 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:03.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.836144 29566 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:03.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.836799 29566 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:03.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.871594 29566 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:03.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.872011 29566 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:03.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.872375 29566 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:03.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.872738 29566 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:03.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.873055 29566 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:03.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.873322 29566 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38709/jwks.json
08:08:03.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.877110 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } attempt: 7
08:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.889598 29489 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } has no permanent_uuid. Determining permanent_uuid...
08:08:03.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.895826 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.918968 29566 master_runner.cc:387] Master server non-default flags:
08:08:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.184.84:36815
08:08:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/data
08:08:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/wal
08:08:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.184.126:39579,127.28.184.125:44265,127.28.184.124:42655
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.184.124:42655
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38709/jwks.json
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/data/info.pb
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.184.124
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/logs
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 07:43:20 UTC on e7f111948823
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12451
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.920454 29564 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 4
08:08:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.920959 29566 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:03.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.923166 29566 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:03.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.925215 29564 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.935741 29578 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:03.954205 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 1
08:08:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.958356 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:04.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:04.039810 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 2
08:08:04.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:04.047405 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:04.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:04.133036 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 3
08:08:04.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:04.141351 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:04.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:04.224619 29564 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 5
08:08:04.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:04.232725 29564 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:04.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:04.289325 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 4
08:08:04.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:04.296824 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:04.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:04.580535 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 5
08:08:04.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:04.589291 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:04.780323 29564 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 6
08:08:04.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:04.792773 29564 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.141989 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 6
08:08:05.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:05.152467 29489 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.184.124:42655: Network error: Client connection negotiation failed: client connection to 127.28.184.124:42655: connect: Connection refused (error 111)
08:08:05.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:03.936305 29579 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:05.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:05.233319 29581 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:05.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:05.235752 29580 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1293 milliseconds
08:08:05.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.235862 29566 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:05.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.237298 29566 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.240155 29566 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:05.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.241526 29566 hybrid_clock.cc:648] HybridClock initialized: now 1781770085241470 us; error 69 us; skew 500 ppm
08:08:05.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.242364 29566 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:05.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.249954 29566 webserver.cc:533] Webserver started at http://127.28.184.124:35633/ using document root <none> and password file <none>
08:08:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.250914 29566 fs_manager.cc:362] Metadata directory not provided
08:08:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.251106 29566 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.251540 29566 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:08:05.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.256290 29566 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/data/instance:
08:08:05.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fb0638f12a734ccfa7ffedb4f429d882"
08:08:05.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:05 on dist-test-slave-d25z"
08:08:05.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.257544 29566 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/wal/instance:
08:08:05.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fb0638f12a734ccfa7ffedb4f429d882"
08:08:05.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:05 on dist-test-slave-d25z"
08:08:05.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.265524 29566 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
08:08:05.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.271750 29592 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:05.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.273061 29566 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.001s
08:08:05.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.273393 29566 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/data,/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/wal
08:08:05.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fb0638f12a734ccfa7ffedb4f429d882"
08:08:05.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:05 on dist-test-slave-d25z"
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.273718 29566 fs_report.cc:389] FS layout report
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/wal
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/wal
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/data/data
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:05.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.327452 29566 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.329107 29566 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.329555 29566 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:05.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.413112 29566 rpc_server.cc:307] RPC server started. Bound to: 127.28.184.124:42655
08:08:05.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.413237 29643 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.184.124:42655 every 8 connection(s)
08:08:05.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.416251 29566 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/data/info.pb
08:08:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.418946 29409 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu as pid 29566
08:08:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.419544 29409 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/master-2/wal/instance
08:08:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.423312 29644 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:08:05.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.441834 29644 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } has no permanent_uuid. Determining permanent_uuid...
08:08:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.468750 29644 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } has no permanent_uuid. Determining permanent_uuid...
08:08:05.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.481560 29644 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } has no permanent_uuid. Determining permanent_uuid...
08:08:05.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.503238 29644 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882: Bootstrap starting.
08:08:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.511565 29644 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882: Neither blocks nor log segments found. Creating new log.
08:08:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.513317 29644 log.cc:826] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882: Log is configured to *not* fsync() on all Append() calls
08:08:05.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.518440 29644 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882: No bootstrap required, opened a new log
08:08:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.539175 29644 raft_consensus.cc:359] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } }
08:08:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.540161 29644 raft_consensus.cc:385] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:08:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.540504 29644 raft_consensus.cc:740] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fb0638f12a734ccfa7ffedb4f429d882, State: Initialized, Role: FOLLOWER
08:08:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.541579 29644 consensus_queue.cc:260] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } }
08:08:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.544586 29653 sys_catalog.cc:455] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } } }
08:08:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.545288 29653 sys_catalog.cc:458] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [sys.catalog]: This master's current role is: FOLLOWER
08:08:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.546320 29644 sys_catalog.cc:565] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [sys.catalog]: configured and running, proceeding with master startup.
08:08:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:05.575686 29664 catalog_manager.cc:1593] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:05.576037 29664 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:05.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:05.585219 29598 tablet.cc:2406] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:08:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.568733 29566 master_runner.cc:187] Error getting master registration for 127.28.184.126:39579: OK, instance_id { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" instance_seqno: 1781770081469636 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:08:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.594591 29566 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:08:05.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.630370 29653 raft_consensus.cc:493] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:05.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.630848 29653 raft_consensus.cc:515] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } }
08:08:05.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.633232 29653 leader_election.cc:290] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f538a1f9c19b473a9c7aff0155f0fc71 (127.28.184.126:39579), 29d7f53fb1244eed8714a8801297d276 (127.28.184.125:44265)
08:08:05.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.853434 29564 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 7
08:08:05.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.894131 29564 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276: Bootstrap starting.
08:08:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.901542 29564 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276: Neither blocks nor log segments found. Creating new log.
08:08:05.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.903743 29564 log.cc:826] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276: Log is configured to *not* fsync() on all Append() calls
08:08:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.911067 29564 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276: No bootstrap required, opened a new log
08:08:05.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.938862 29564 raft_consensus.cc:359] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } }
08:08:05.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.940008 29564 raft_consensus.cc:385] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:08:05.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.940305 29564 raft_consensus.cc:740] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 29d7f53fb1244eed8714a8801297d276, State: Initialized, Role: FOLLOWER
08:08:05.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.941290 29564 consensus_queue.cc:260] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } }
08:08:05.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.944274 29668 sys_catalog.cc:455] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } } }
08:08:05.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.945226 29668 sys_catalog.cc:458] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [sys.catalog]: This master's current role is: FOLLOWER
08:08:05.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.946943 29564 sys_catalog.cc:565] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [sys.catalog]: configured and running, proceeding with master startup.
08:08:05.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.947182 29539 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fb0638f12a734ccfa7ffedb4f429d882" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "29d7f53fb1244eed8714a8801297d276" is_pre_election: true
08:08:05.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.948192 29539 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fb0638f12a734ccfa7ffedb4f429d882 in term 0.
08:08:05.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.949776 29596 leader_election.cc:304] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 29d7f53fb1244eed8714a8801297d276, fb0638f12a734ccfa7ffedb4f429d882; no voters:
08:08:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.950765 29653 raft_consensus.cc:2804] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:08:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.951177 29653 raft_consensus.cc:493] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:08:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.951530 29653 raft_consensus.cc:3060] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [term 0 FOLLOWER]: Advancing to term 1
08:08:05.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.959995 29653 raft_consensus.cc:515] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } }
08:08:05.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.962183 29653 leader_election.cc:290] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [CANDIDATE]: Term 1 election: Requested vote from peers f538a1f9c19b473a9c7aff0155f0fc71 (127.28.184.126:39579), 29d7f53fb1244eed8714a8801297d276 (127.28.184.125:44265)
08:08:05.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.963932 29539 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fb0638f12a734ccfa7ffedb4f429d882" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "29d7f53fb1244eed8714a8801297d276"
08:08:05.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.964568 29539 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [term 0 FOLLOWER]: Advancing to term 1
08:08:05.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.979229 29539 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fb0638f12a734ccfa7ffedb4f429d882 in term 1.
08:08:05.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.980417 29596 leader_election.cc:304] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 29d7f53fb1244eed8714a8801297d276, fb0638f12a734ccfa7ffedb4f429d882; no voters:
08:08:05.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.981259 29653 raft_consensus.cc:2804] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [term 1 FOLLOWER]: Leader election won for term 1
08:08:05.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.983764 29653 raft_consensus.cc:697] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [term 1 LEADER]: Becoming Leader. State: Replica: fb0638f12a734ccfa7ffedb4f429d882, State: Running, Role: LEADER
08:08:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.984936 29653 consensus_queue.cc:237] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } }
08:08:06.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:05.999307 29676 sys_catalog.cc:455] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fb0638f12a734ccfa7ffedb4f429d882. Latest consensus state: current_term: 1 leader_uuid: "fb0638f12a734ccfa7ffedb4f429d882" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } } }
08:08:06.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.000089 29676 sys_catalog.cc:458] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [sys.catalog]: This master's current role is: LEADER
08:08:06.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.008335 29680 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
08:08:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.017144 29680 catalog_manager.cc:1519] Initializing Kudu cluster ID...
08:08:06.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:06.036262 29682 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:06.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:06.036962 29682 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:06.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.046126 29539 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [term 1 FOLLOWER]: Refusing update from remote peer fb0638f12a734ccfa7ffedb4f429d882: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:08:06.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.049783 29676 consensus_queue.cc:1048] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [LEADER]: Connected to new peer: Peer: permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:08:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.017726 29491 master_runner.cc:187] Error getting master registration for 127.28.184.126:39579: OK, instance_id { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" instance_seqno: 1781770081469636 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:08:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.058889 29491 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:08:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.095858 29668 sys_catalog.cc:455] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fb0638f12a734ccfa7ffedb4f429d882. Latest consensus state: current_term: 1 leader_uuid: "fb0638f12a734ccfa7ffedb4f429d882" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } } }
08:08:06.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.100668 29668 sys_catalog.cc:458] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [sys.catalog]: This master's current role is: FOLLOWER
08:08:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.107108 29684 mvcc.cc:204] Tried to move back new op lower bound from 7298130272406941696 to 7298130272205680640. Current Snapshot: MvccSnapshot[applied={T|T < 7298130272406941696}]
08:08:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.109797 29653 sys_catalog.cc:455] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fb0638f12a734ccfa7ffedb4f429d882" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } } }
08:08:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.110581 29653 sys_catalog.cc:458] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [sys.catalog]: This master's current role is: LEADER
08:08:06.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.115550 29680 catalog_manager.cc:1382] Generated new cluster ID: 0da8e190922141bcad5a9f54f105af7f
08:08:06.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.115847 29680 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
08:08:06.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.135721 29668 sys_catalog.cc:455] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fb0638f12a734ccfa7ffedb4f429d882" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } } }
08:08:06.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.136504 29668 sys_catalog.cc:458] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276 [sys.catalog]: This master's current role is: FOLLOWER
08:08:06.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.158349 29680 catalog_manager.cc:1405] Generated new certificate authority record
08:08:06.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.160392 29680 catalog_manager.cc:1539] Loading token signing keys...
08:08:06.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.185993 29489 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } attempt: 7
08:08:06.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.188078 29680 catalog_manager.cc:6069] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882: Generated new TSK 0
08:08:06.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.190858 29680 catalog_manager.cc:1549] Initializing in-progress tserver states...
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.223901 29409 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/wal
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/data
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/logs
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/data/info.pb
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:08:06.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.184.65:0
08:08:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.184.65
08:08:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.184.65
08:08:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.184.126:39579,127.28.184.125:44265,127.28.184.124:42655
08:08:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.184.84:36815
08:08:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:08:06.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.231248 29489 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71: Bootstrap starting.
08:08:06.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.240978 29489 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71: Neither blocks nor log segments found. Creating new log.
08:08:06.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.243815 29489 log.cc:826] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71: Log is configured to *not* fsync() on all Append() calls
08:08:06.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.256125 29489 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71: No bootstrap required, opened a new log
08:08:06.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.292505 29489 raft_consensus.cc:359] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } }
08:08:06.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.293723 29489 raft_consensus.cc:385] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:08:06.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.294212 29489 raft_consensus.cc:740] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f538a1f9c19b473a9c7aff0155f0fc71, State: Initialized, Role: FOLLOWER
08:08:06.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.295536 29489 consensus_queue.cc:260] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } }
08:08:06.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.304159 29693 sys_catalog.cc:455] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } } }
08:08:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.308043 29489 sys_catalog.cc:565] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [sys.catalog]: configured and running, proceeding with master startup.
08:08:06.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.309008 29463 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fb0638f12a734ccfa7ffedb4f429d882" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f538a1f9c19b473a9c7aff0155f0fc71"
08:08:06.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.309755 29463 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [term 0 FOLLOWER]: Advancing to term 1
08:08:06.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.308445 29693 sys_catalog.cc:458] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [sys.catalog]: This master's current role is: FOLLOWER
08:08:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.308228 29464 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fb0638f12a734ccfa7ffedb4f429d882" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" is_pre_election: true
08:08:06.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.322634 29463 raft_consensus.cc:2468] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fb0638f12a734ccfa7ffedb4f429d882 in term 1.
08:08:06.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.332624 29462 raft_consensus.cc:1275] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [term 1 FOLLOWER]: Refusing update from remote peer fb0638f12a734ccfa7ffedb4f429d882: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:08:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.342686 29653 consensus_queue.cc:1048] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
08:08:06.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.398556 29693 sys_catalog.cc:455] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fb0638f12a734ccfa7ffedb4f429d882. Latest consensus state: current_term: 1 leader_uuid: "fb0638f12a734ccfa7ffedb4f429d882" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } } }
08:08:06.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.399394 29693 sys_catalog.cc:458] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [sys.catalog]: This master's current role is: FOLLOWER
08:08:06.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.406137 29706 sys_catalog.cc:455] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "fb0638f12a734ccfa7ffedb4f429d882" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } } }
08:08:06.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.409610 29706 sys_catalog.cc:458] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71 [sys.catalog]: This master's current role is: FOLLOWER
08:08:06.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.412186 29708 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:08:06.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.795646 29653 sys_catalog.cc:455] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "fb0638f12a734ccfa7ffedb4f429d882" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f538a1f9c19b473a9c7aff0155f0fc71" member_type: VOTER last_known_addr { host: "127.28.184.126" port: 39579 } } peers { permanent_uuid: "29d7f53fb1244eed8714a8801297d276" member_type: VOTER last_known_addr { host: "127.28.184.125" port: 44265 } } peers { permanent_uuid: "fb0638f12a734ccfa7ffedb4f429d882" member_type: VOTER last_known_addr { host: "127.28.184.124" port: 42655 } } }
08:08:06.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:06.796639 29653 sys_catalog.cc:458] T 00000000000000000000000000000000 P fb0638f12a734ccfa7ffedb4f429d882 [sys.catalog]: This master's current role is: LEADER
08:08:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:06.879616 29691 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:06.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:06.880398 29691 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:06.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:06.881184 29691 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:06.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:06.944891 29691 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:06.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:06.946297 29691 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.184.65
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:07.013517 29691 tablet_server_runner.cc:78] Tablet server non-default flags:
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.184.84:36815
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/data
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/wal
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.184.65:0
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/data/info.pb
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.184.65
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.184.126:39579,127.28.184.125:44265,127.28.184.124:42655
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.184.65
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/logs
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 07:43:20 UTC on e7f111948823
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12451
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:07.015520 29691 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:07.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:07.017915 29691 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:07.034924 29716 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:07.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:07.044883 29682 catalog_manager.cc:1294] Loaded cluster ID: 0da8e190922141bcad5a9f54f105af7f
08:08:07.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:07.045558 29682 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276: loading cluster ID for follower catalog manager: success
08:08:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:07.055693 29682 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276: acquiring CA information for follower catalog manager: success
08:08:07.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:07.076154 29682 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 29d7f53fb1244eed8714a8801297d276: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:07.442399 29708 catalog_manager.cc:1294] Loaded cluster ID: 0da8e190922141bcad5a9f54f105af7f
08:08:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:07.444159 29708 catalog_manager.cc:1587] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71: loading cluster ID for follower catalog manager: success
08:08:07.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:07.468246 29708 catalog_manager.cc:1609] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71: acquiring CA information for follower catalog manager: success
08:08:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:07.511910 29708 catalog_manager.cc:1637] T 00000000000000000000000000000000 P f538a1f9c19b473a9c7aff0155f0fc71: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:08:09.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:08.437793 29715 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 29691
08:08:09.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:09.508837 29715 kernel_stack_watchdog.cc:198] Thread 29691 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
08:08:09.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:08:09.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:08:09.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:08:09.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:08:09.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:07.038522 29717 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:09.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:09.512763 29691 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.476s user 0.696s sys 1.214s
08:08:09.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:09.513130 29691 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.477s user 0.696s sys 1.214s
08:08:09.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:09.523679 29724 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.525513 29691 server_base.cc:1061] running on GCE node
08:08:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.527154 29691 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:09.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.544497 29691 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.546098 29691 hybrid_clock.cc:648] HybridClock initialized: now 1781770089545797 us; error 310 us; skew 500 ppm
08:08:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.547259 29691 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.558630 29691 webserver.cc:533] Webserver started at http://127.28.184.65:46873/ using document root <none> and password file <none>
08:08:09.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.559934 29691 fs_manager.cc:362] Metadata directory not provided
08:08:09.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.560213 29691 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:09.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.560803 29691 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:08:09.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.567127 29691 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/data/instance:
08:08:09.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25f7d5c7aa394bb4918ccdb31c19f5dd"
08:08:09.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:09 on dist-test-slave-d25z"
08:08:09.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.568614 29691 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/wal/instance:
08:08:09.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25f7d5c7aa394bb4918ccdb31c19f5dd"
08:08:09.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:09 on dist-test-slave-d25z"
08:08:09.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.578672 29691 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.008s sys 0.000s
08:08:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.585206 29729 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.586547 29691 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
08:08:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.586967 29691 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/data,/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/wal
08:08:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25f7d5c7aa394bb4918ccdb31c19f5dd"
08:08:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:09 on dist-test-slave-d25z"
08:08:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.587364 29691 fs_report.cc:389] FS layout report
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/wal
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/wal
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/data/data
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.645957 29691 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:09.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.647876 29691 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:09.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.648437 29691 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.652451 29691 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:08:09.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.657773 29691 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:08:09.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.658013 29691 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:08:09.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.658347 29691 ts_tablet_manager.cc:616] Registered 0 tablets
08:08:09.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.658540 29691 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:08:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.858877 29691 rpc_server.cc:307] RPC server started. Bound to: 127.28.184.65:40389
08:08:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.859004 29841 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.184.65:40389 every 8 connection(s)
08:08:09.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.861913 29691 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/data/info.pb
08:08:09.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.866375 29409 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu as pid 29691
08:08:09.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.867091 29409 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-0/wal/instance
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.893783 29409 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/wal
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/data
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/logs
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/data/info.pb
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.184.66:0
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.184.66
08:08:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.184.66
08:08:09.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:09.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.184.126:39579,127.28.184.125:44265,127.28.184.124:42655
08:08:09.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.184.84:36815
08:08:09.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:09.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:09.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:08:09.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.916288 29843 heartbeater.cc:344] Connected to a master server at 127.28.184.124:42655
08:08:09.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.916917 29843 heartbeater.cc:461] Registering TS with master...
08:08:09.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.918262 29844 heartbeater.cc:344] Connected to a master server at 127.28.184.126:39579
08:08:09.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.918529 29843 heartbeater.cc:507] Master 127.28.184.124:42655 requested a full tablet report, sending...
08:08:09.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.918650 29844 heartbeater.cc:461] Registering TS with master...
08:08:09.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.919778 29844 heartbeater.cc:507] Master 127.28.184.126:39579 requested a full tablet report, sending...
08:08:09.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.922292 29609 ts_manager.cc:194] Registered new tserver with Master: 25f7d5c7aa394bb4918ccdb31c19f5dd (127.28.184.65:40389)
08:08:09.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.925809 29609 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.184.65:42541
08:08:09.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.925827 29454 ts_manager.cc:194] Registered new tserver with Master: 25f7d5c7aa394bb4918ccdb31c19f5dd (127.28.184.65:40389)
08:08:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.934743 29842 heartbeater.cc:344] Connected to a master server at 127.28.184.125:44265
08:08:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.935143 29842 heartbeater.cc:461] Registering TS with master...
08:08:09.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.935851 29842 heartbeater.cc:507] Master 127.28.184.125:44265 requested a full tablet report, sending...
08:08:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:09.939275 29529 ts_manager.cc:194] Registered new tserver with Master: 25f7d5c7aa394bb4918ccdb31c19f5dd (127.28.184.65:40389)
08:08:10.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:10.265579 29852 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:10.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:10.266244 29852 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:10.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:10.266824 29852 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:10.302647 29852 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:10.303683 29852 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.184.66
08:08:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:10.345341 29852 tablet_server_runner.cc:78] Tablet server non-default flags:
08:08:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.184.84:36815
08:08:10.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/data
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/wal
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.184.66:0
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/data/info.pb
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.184.66
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.184.126:39579,127.28.184.125:44265,127.28.184.124:42655
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.184.66
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/logs
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 07:43:20 UTC on e7f111948823
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12451
08:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:10.347271 29852 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:10.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:10.349387 29852 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:10.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:10.364856 29858 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:10.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:10.930617 29843 heartbeater.cc:499] Master 127.28.184.124:42655 was elected leader, sending a full tablet report...
08:08:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:11.768972 29857 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 29852
08:08:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:10.366400 29859 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:12.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:12.212191 29852 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.846s user 0.747s sys 1.063s
08:08:12.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:12.212978 29852 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.846s user 0.748s sys 1.063s
08:08:12.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:12.215242 29861 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:12.218147 29860 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1846 milliseconds
08:08:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.218226 29852 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:12.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.219642 29852 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:12.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.221889 29852 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:12.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.223281 29852 hybrid_clock.cc:648] HybridClock initialized: now 1781770092223221 us; error 70 us; skew 500 ppm
08:08:12.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.224408 29852 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:12.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.231144 29852 webserver.cc:533] Webserver started at http://127.28.184.66:45787/ using document root <none> and password file <none>
08:08:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.232112 29852 fs_manager.cc:362] Metadata directory not provided
08:08:12.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.232378 29852 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:12.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.232860 29852 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:08:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.237778 29852 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/data/instance:
08:08:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c010ae4894845119e3a41d9632e26a8"
08:08:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:12 on dist-test-slave-d25z"
08:08:12.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.240564 29852 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/wal/instance:
08:08:12.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c010ae4894845119e3a41d9632e26a8"
08:08:12.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:12 on dist-test-slave-d25z"
08:08:12.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.249500 29852 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
08:08:12.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.255692 29869 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:12.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.256981 29852 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
08:08:12.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.257345 29852 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/data,/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/wal
08:08:12.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2c010ae4894845119e3a41d9632e26a8"
08:08:12.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:12 on dist-test-slave-d25z"
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.257704 29852 fs_report.cc:389] FS layout report
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/wal
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/wal
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/data/data
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:12.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.313220 29852 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.314785 29852 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.315263 29852 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.318010 29852 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:08:12.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.323083 29852 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:08:12.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.323330 29852 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:08:12.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.323652 29852 ts_tablet_manager.cc:616] Registered 0 tablets
08:08:12.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.323849 29852 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:08:12.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.476109 29852 rpc_server.cc:307] RPC server started. Bound to: 127.28.184.66:43589
08:08:12.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.476236 29981 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.184.66:43589 every 8 connection(s)
08:08:12.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.479635 29852 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/data/info.pb
08:08:12.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.480994 29409 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu as pid 29852
08:08:12.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.481490 29409 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-1/wal/instance
08:08:12.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.493068 29409 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:12.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:12.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/wal
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/data
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/logs
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/data/info.pb
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.184.67:0
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.184.67
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.184.67
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.184.126:39579,127.28.184.125:44265,127.28.184.124:42655
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.184.84:36815
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:12.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:08:12.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.539248 29983 heartbeater.cc:344] Connected to a master server at 127.28.184.124:42655
08:08:12.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.539767 29983 heartbeater.cc:461] Registering TS with master...
08:08:12.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.542903 29983 heartbeater.cc:507] Master 127.28.184.124:42655 requested a full tablet report, sending...
08:08:12.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.545977 29609 ts_manager.cc:194] Registered new tserver with Master: 2c010ae4894845119e3a41d9632e26a8 (127.28.184.66:43589)
08:08:12.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.547976 29609 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.184.66:47681
08:08:12.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.560326 29982 heartbeater.cc:344] Connected to a master server at 127.28.184.125:44265
08:08:12.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.563403 29982 heartbeater.cc:461] Registering TS with master...
08:08:12.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.564036 29982 heartbeater.cc:507] Master 127.28.184.125:44265 requested a full tablet report, sending...
08:08:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.567158 29529 ts_manager.cc:194] Registered new tserver with Master: 2c010ae4894845119e3a41d9632e26a8 (127.28.184.66:43589)
08:08:12.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.569067 29989 heartbeater.cc:344] Connected to a master server at 127.28.184.126:39579
08:08:12.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.569419 29989 heartbeater.cc:461] Registering TS with master...
08:08:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.570040 29989 heartbeater.cc:507] Master 127.28.184.126:39579 requested a full tablet report, sending...
08:08:12.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.572791 29454 ts_manager.cc:194] Registered new tserver with Master: 2c010ae4894845119e3a41d9632e26a8 (127.28.184.66:43589)
08:08:12.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:12.877275 29988 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:12.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:12.877949 29988 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:12.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:12.878479 29988 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:12.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:12.915154 29988 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:12.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:12.916029 29988 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.184.67
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.960435 29988 tablet_server_runner.cc:78] Tablet server non-default flags:
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.184.84:36815
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/data
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/wal
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.184.67:0
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/data/info.pb
08:08:12.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.184.67
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.184.126:39579,127.28.184.125:44265,127.28.184.124:42655
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.184.67
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/logs
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 07:43:20 UTC on e7f111948823
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12451
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:12.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.962589 29988 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:12.964744 29988 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:12.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:12.978770 29998 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:13.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:13.552232 29983 heartbeater.cc:499] Master 127.28.184.124:42655 was elected leader, sending a full tablet report...
08:08:14.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:12.979466 29999 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:14.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:14.383520 29997 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 29988
08:08:14.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:14.715916 29988 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.737s user 0.556s sys 1.085s
08:08:14.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:14.717796 29988 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.739s user 0.556s sys 1.085s
08:08:14.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:14.718518 30001 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:14.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:14.722285 30000 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1738 milliseconds
08:08:14.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.722311 29988 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:14.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.723626 29988 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.726696 29988 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:14.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.728173 29988 hybrid_clock.cc:648] HybridClock initialized: now 1781770094728140 us; error 40 us; skew 500 ppm
08:08:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.729058 29988 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.735672 29988 webserver.cc:533] Webserver started at http://127.28.184.67:45333/ using document root <none> and password file <none>
08:08:14.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.736749 29988 fs_manager.cc:362] Metadata directory not provided
08:08:14.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.736992 29988 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:14.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.737526 29988 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:08:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.742380 29988 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/data/instance:
08:08:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "64595f2578a94822836d0a8dd3593e71"
08:08:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:14 on dist-test-slave-d25z"
08:08:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.743562 29988 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/wal/instance:
08:08:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "64595f2578a94822836d0a8dd3593e71"
08:08:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:14 on dist-test-slave-d25z"
08:08:14.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.751554 29988 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
08:08:14.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.757807 30009 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:14.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.758932 29988 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
08:08:14.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.759269 29988 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/data,/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/wal
08:08:14.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "64595f2578a94822836d0a8dd3593e71"
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:14 on dist-test-slave-d25z"
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.759613 29988 fs_report.cc:389] FS layout report
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/wal
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/wal
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/data/data
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:14.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.842475 29988 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:14.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.844240 29988 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:14.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.844764 29988 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:14.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.847831 29988 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:08:14.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.852530 29988 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:08:14.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.852766 29988 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
08:08:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.853076 29988 ts_tablet_manager.cc:616] Registered 0 tablets
08:08:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:14.853268 29988 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:08:15.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.015951 29988 rpc_server.cc:307] RPC server started. Bound to: 127.28.184.67:36699
08:08:15.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.016089 30121 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.184.67:36699 every 8 connection(s)
08:08:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.019186 29988 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/data/info.pb
08:08:15.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.023751 29409 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu as pid 29988
08:08:15.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.024268 29409 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster8860893194126431458/ts-2/wal/instance
08:08:15.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.059716 30124 heartbeater.cc:344] Connected to a master server at 127.28.184.126:39579
08:08:15.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.059911 30123 heartbeater.cc:344] Connected to a master server at 127.28.184.124:42655
08:08:15.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.060333 30123 heartbeater.cc:461] Registering TS with master...
08:08:15.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.060333 30124 heartbeater.cc:461] Registering TS with master...
08:08:15.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.061204 30122 heartbeater.cc:344] Connected to a master server at 127.28.184.125:44265
08:08:15.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.061544 30122 heartbeater.cc:461] Registering TS with master...
08:08:15.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.062136 30124 heartbeater.cc:507] Master 127.28.184.126:39579 requested a full tablet report, sending...
08:08:15.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.062137 30123 heartbeater.cc:507] Master 127.28.184.124:42655 requested a full tablet report, sending...
08:08:15.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.062485 30122 heartbeater.cc:507] Master 127.28.184.125:44265 requested a full tablet report, sending...
08:08:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.066099 29454 ts_manager.cc:194] Registered new tserver with Master: 64595f2578a94822836d0a8dd3593e71 (127.28.184.67:36699)
08:08:15.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.068791 29609 ts_manager.cc:194] Registered new tserver with Master: 64595f2578a94822836d0a8dd3593e71 (127.28.184.67:36699)
08:08:15.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.070765 29609 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.184.67:38801
08:08:15.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.071801 29529 ts_manager.cc:194] Registered new tserver with Master: 64595f2578a94822836d0a8dd3593e71 (127.28.184.67:36699)
08:08:15.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.079423 29409 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
08:08:15.092 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:08:15.092 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:08:15.093 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.28.184.126"
port: 39579
}
bound_http_address {
host: "127.28.184.126"
port: 39989
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.28.184.125"
port: 44265
}
bound_http_address {
host: "127.28.184.125"
port: 39417
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.28.184.124"
port: 42655
}
bound_http_address {
host: "127.28.184.124"
port: 35633
}
}
}
08:08:15.094 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:08:15.095 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.28.184.65"
port: 40389
}
bound_http_address {
host: "127.28.184.65"
port: 46873
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.28.184.66"
port: 43589
}
bound_http_address {
host: "127.28.184.66"
port: 45787
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.28.184.67"
port: 36699
}
bound_http_address {
host: "127.28.184.67"
port: 45333
}
}
}
08:08:15.095 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:08:15.096 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster3045996094370665746/krb5kdc/krb5cc
08:08:15.097 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:08:15.098 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskbTrDWn/test-tmp
08:08:15.098 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:08:15.098 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskbTrDWn/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:08:15.101 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
08:08:15.470 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:15.470 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:08:15.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.474449 30133 mini_oidc.cc:150] Starting JWKS server
08:08:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.479133 30133 webserver.cc:533] Webserver started at http://127.0.0.1:34573/ using document root <none> and password file <none>
08:08:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.479739 30133 mini_oidc.cc:186] Starting OIDC Discovery server
08:08:15.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:15.482280 30133 webserver.cc:533] Webserver started at http://127.0.0.1:33455/ using document root <none> and password file <none>
08:08:15.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:08:15.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/principal' for realm 'KRBTEST.COM',
08:08:15.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:08:15.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:15 dist-test-slave-d25z krb5kdc[30144](info): setting up network...
08:08:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:08:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:15 dist-test-slave-d25z krb5kdc[30144](info): set up 2 sockets
08:08:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:15 dist-test-slave-d25z krb5kdc[30144](info): commencing operation
08:08:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:08:16.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:16.074807 30123 heartbeater.cc:499] Master 127.28.184.124:42655 was elected leader, sending a full tablet report...
08:08:17.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:17.656598 30133 mini_kdc.cc:121] Time spent starting KDC: real 2.126s user 0.001s sys 0.023s
08:08:17.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:08:17.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:08:17.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:08:17.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:08:17.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:08:17.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:08:17.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:08:17.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:08:17.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:08:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:08:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/test-user.keytab.
08:08:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/test-user.keytab.
08:08:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:17 dist-test-slave-d25z krb5kdc[30144](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770097, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:08:17.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-18T08:08:17Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:08:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-18T08:08:17Z Disabled control of system clock
08:08:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.109.126@KRBTEST.COM; defaulting to no policy
08:08:17.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:17.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.109.126@KRBTEST.COM" created.
08:08:17.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:17.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.109.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab.
08:08:17.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.109.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab.
08:08:17.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.29.109.126@KRBTEST.COM; defaulting to no policy
08:08:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:17.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.29.109.126@KRBTEST.COM" created.
08:08:17.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:17.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.109.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab.
08:08:17.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.109.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab.
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:17.925993 30133 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/logs
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data/info.pb
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:17.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.109.126:34513
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.109.126
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.109.126
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.109.84:43249
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34573/jwks.json
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:17.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:18.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:18.291793 30161 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:18.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:18.292642 30161 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:18.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:18.293462 30161 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:18.329141 30161 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:18.329478 30161 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:18.329864 30161 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:18.330186 30161 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:18.330495 30161 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:18.330801 30161 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:18.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:18.331048 30161 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34573/jwks.json
08:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:18.374171 30161 master_runner.cc:387] Master server non-default flags:
08:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.109.84:43249
08:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data
08:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal
08:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405
08:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab
08:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.109.126
08:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.109.126:34513
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34573/jwks.json
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data/info.pb
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.109.126
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/logs
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 07:43:20 UTC on e7f111948823
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12451
08:08:18.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:18.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:18.376017 30161 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:18.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:18.377231 30161 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:18.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:18.390313 30167 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:19.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:18.394979 30170 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:18.391889 30168 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:19.759563 30169 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1365 milliseconds
08:08:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.759683 30161 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:19.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.761130 30161 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:19.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.764282 30161 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.765740 30161 hybrid_clock.cc:648] HybridClock initialized: now 1781770099765674 us; error 77 us; skew 500 ppm
08:08:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.766584 30161 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:19.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:19 dist-test-slave-d25z krb5kdc[30144](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770099, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:19.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.773458 30161 init.cc:377] Logged in from keytab as kudu/127.29.109.126@KRBTEST.COM (short username kudu)
08:08:19.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.779341 30161 webserver.cc:533] Webserver started at http://127.29.109.126:43499/ using document root <none> and password file <none>
08:08:19.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.780284 30161 fs_manager.cc:362] Metadata directory not provided
08:08:19.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.780516 30161 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:19.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.780987 30161 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:08:19.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.785604 30161 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data/instance:
08:08:19.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2efcd7c2409142d18d168ab8cfbc2d72"
08:08:19.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:19 on dist-test-slave-d25z"
08:08:19.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.786695 30161 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal/instance:
08:08:19.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2efcd7c2409142d18d168ab8cfbc2d72"
08:08:19.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:19 on dist-test-slave-d25z"
08:08:19.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.794973 30161 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.003s
08:08:19.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.801131 30180 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:19.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.802383 30161 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.802760 30161 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data,/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2efcd7c2409142d18d168ab8cfbc2d72"
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:19 on dist-test-slave-d25z"
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.803087 30161 fs_report.cc:389] FS layout report
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data/data
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:19.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.873327 30161 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:19.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.879760 30161 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:19.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.880239 30161 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.970613 30161 rpc_server.cc:307] RPC server started. Bound to: 127.29.109.126:34513
08:08:19.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.970670 30231 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.109.126:34513 every 8 connection(s)
08:08:19.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.973733 30161 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data/info.pb
08:08:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.975126 30133 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu as pid 30161
08:08:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.975649 30133 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal/instance
08:08:19.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.983129 30232 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:08:19.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.109.125@KRBTEST.COM; defaulting to no policy
08:08:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:19.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.109.125@KRBTEST.COM" created.
08:08:20.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:19.997304 30232 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } has no permanent_uuid. Determining permanent_uuid...
08:08:20.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:20.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.109.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab.
08:08:20.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.109.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab.
08:08:20.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:20 dist-test-slave-d25z krb5kdc[30144](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770099, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.126@KRBTEST.COM for kudu/127.29.109.126@KRBTEST.COM
08:08:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.29.109.125@KRBTEST.COM; defaulting to no policy
08:08:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:20.044255 30232 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } has no permanent_uuid. Determining permanent_uuid...
08:08:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.29.109.125@KRBTEST.COM" created.
08:08:20.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.047180 30182 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.052368 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.125:38999: Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.109.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab.
08:08:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.109.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab.
08:08:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:20.070631 30133 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/logs
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data/info.pb
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.109.125:38999
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.109.125
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.109.125
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.109.84:43249
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34573/jwks.json
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:20.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:20.101994 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } attempt: 1
08:08:20.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.106779 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.125:38999: Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:20.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:20.175246 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } attempt: 2
08:08:20.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.180725 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.125:38999: Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:20.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:20.272181 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } attempt: 3
08:08:20.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.277551 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.125:38999: Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:20.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.414958 30241 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:20.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.415843 30241 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:20.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.416939 30241 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:20.420945 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } attempt: 4
08:08:20.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.425136 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.125:38999: Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:20.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.464745 30241 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:20.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.465116 30241 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:20.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.465521 30241 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:20.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.465857 30241 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:20.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.466173 30241 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:20.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.466487 30241 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:20.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.466745 30241 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34573/jwks.json
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:20.512521 30241 master_runner.cc:387] Master server non-default flags:
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.109.84:43249
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.109.125
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.109.125:38999
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34573/jwks.json
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data/info.pb
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.109.125
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/logs
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 07:43:20 UTC on e7f111948823
08:08:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12451
08:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:20.514393 30241 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:20.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:20.515713 30241 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:20.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.529290 30248 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:20.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:20.728569 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } attempt: 5
08:08:20.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.737661 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.125:38999: Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:21.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:21.285280 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } attempt: 6
08:08:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:21.318930 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.125:38999: Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:22.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:21.948192 30247 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 30241
08:08:22.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:20.529896 30249 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:22.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.186339 30247 kernel_stack_watchdog.cc:198] Thread 30241 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 415ms:
08:08:22.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:08:22.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:08:22.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:08:22.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:08:22.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.192732 30241 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.663s user 0.005s sys 0.004s
08:08:22.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.193035 30250 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1655 milliseconds
08:08:22.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.193229 30241 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.664s user 0.005s sys 0.004s
08:08:22.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.194826 30251 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:22.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.194923 30241 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:22.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.196372 30241 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:22.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.198868 30241 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:22.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.200273 30241 hybrid_clock.cc:648] HybridClock initialized: now 1781770102200242 us; error 41 us; skew 500 ppm
08:08:22.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.201161 30241 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:22.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:22 dist-test-slave-d25z krb5kdc[30144](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770102, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:22.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.207381 30241 init.cc:377] Logged in from keytab as kudu/127.29.109.125@KRBTEST.COM (short username kudu)
08:08:22.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.212913 30241 webserver.cc:533] Webserver started at http://127.29.109.125:33569/ using document root <none> and password file <none>
08:08:22.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.213817 30241 fs_manager.cc:362] Metadata directory not provided
08:08:22.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.214053 30241 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:22.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.214607 30241 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:08:22.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.218952 30241 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data/instance:
08:08:22.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5cf4af86dd6f4f69aea3307291ff88e6"
08:08:22.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:22 on dist-test-slave-d25z"
08:08:22.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.220077 30241 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal/instance:
08:08:22.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5cf4af86dd6f4f69aea3307291ff88e6"
08:08:22.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:22 on dist-test-slave-d25z"
08:08:22.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.228178 30241 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
08:08:22.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.233946 30262 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:22.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.235412 30241 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.235764 30241 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data,/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5cf4af86dd6f4f69aea3307291ff88e6"
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:22 on dist-test-slave-d25z"
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.236094 30241 fs_report.cc:389] FS layout report
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data/data
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:22.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:22.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:22.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.291488 30241 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:22.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.301690 30241 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:22.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.302189 30241 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:22.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.379608 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } attempt: 7
08:08:22.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.381636 30241 rpc_server.cc:307] RPC server started. Bound to: 127.29.109.125:38999
08:08:22.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.381726 30313 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.109.125:38999 every 8 connection(s)
08:08:22.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.386175 30241 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data/info.pb
08:08:22.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.390540 30133 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu as pid 30241
08:08:22.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.391268 30133 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal/instance
08:08:22.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.395656 30316 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:08:22.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:22 dist-test-slave-d25z krb5kdc[30144](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770099, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.126@KRBTEST.COM for kudu/127.29.109.125@KRBTEST.COM
08:08:22.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.109.124@KRBTEST.COM; defaulting to no policy
08:08:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.109.124@KRBTEST.COM" created.
08:08:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.425644 30232 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } has no permanent_uuid. Determining permanent_uuid...
08:08:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.421484 30316 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } has no permanent_uuid. Determining permanent_uuid...
08:08:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.443336 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:22 dist-test-slave-d25z krb5kdc[30144](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770102, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.125@KRBTEST.COM for kudu/127.29.109.126@KRBTEST.COM
08:08:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.109.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab.
08:08:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.109.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab.
08:08:22.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.471979 30316 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } has no permanent_uuid. Determining permanent_uuid...
08:08:22.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.29.109.124@KRBTEST.COM; defaulting to no policy
08:08:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.29.109.124@KRBTEST.COM" created.
08:08:22.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:22 dist-test-slave-d25z krb5kdc[30144](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770102, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.125@KRBTEST.COM for kudu/127.29.109.125@KRBTEST.COM
08:08:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.501855 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 1
08:08:22.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.504254 30316 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } has no permanent_uuid. Determining permanent_uuid...
08:08:22.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.507682 30265 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:22.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.508776 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:22.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.513271 30316 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:22.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:08:22.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.109.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab.
08:08:22.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.109.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab.
08:08:22.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.517360 30133 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:22.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:22.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/wal
08:08:22.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data
08:08:22.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:22.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:22.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:22.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:22.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:22.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/logs
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data/info.pb
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.109.124:36405
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.109.124
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.109.124
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.109.84:43249
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34573/jwks.json
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:22.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:22.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.562837 30316 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 1
08:08:22.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.568322 30316 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.590221 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 2
08:08:22.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.594159 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:22.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.636878 30316 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 2
08:08:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.641898 30316 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:22.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.679610 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 3
08:08:22.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.683924 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.733464 30316 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 3
08:08:22.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.738703 30316 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:22.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.824502 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 4
08:08:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.829857 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:22.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.882158 30316 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 4
08:08:22.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.882644 30326 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:22.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.883487 30326 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:22.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.884403 30326 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.887424 30316 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:22.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.922376 30326 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:22.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.922683 30326 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.922968 30326 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.923190 30326 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.923465 30326 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:22.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.923722 30326 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.923887 30326 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34573/jwks.json
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.967859 30326 master_runner.cc:387] Master server non-default flags:
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.109.84:43249
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/wal
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.109.124
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.109.124:36405
08:08:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34573/jwks.json
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data/info.pb
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.109.124
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/logs
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:08:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 07:43:20 UTC on e7f111948823
08:08:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12451
08:08:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.969973 30326 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:22.971913 30326 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.987605 30333 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:23.113304 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 5
08:08:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:23.120361 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:23.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:23.196573 30316 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 5
08:08:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:23.204049 30316 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:23.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:23.672925 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 6
08:08:23.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:23.702797 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:23.751595 30316 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 6
08:08:23.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:23.787384 30316 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:24.093423 30310 debug-util.cc:398] Leaking SignalData structure 0x7b0800071060 after lost signal to thread 30243
08:08:24.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:24.094265 30310 debug-util.cc:398] Leaking SignalData structure 0x7b08000717c0 after lost signal to thread 30313
08:08:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:22.988689 30334 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:24.390620 30332 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 30326
08:08:24.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:24.707858 30332 kernel_stack_watchdog.cc:198] Thread 30326 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 394ms:
08:08:24.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:08:24.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:08:24.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:08:24.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:08:24.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:24.713235 30326 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.722s user 0.000s sys 0.003s
08:08:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:24.713703 30335 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1723 milliseconds
08:08:24.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:24.713744 30326 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.722s user 0.000s sys 0.003s
08:08:24.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.716380 30326 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:24.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:24.716382 30337 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:24.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.718006 30326 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:24.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.720989 30326 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:24.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.722500 30326 hybrid_clock.cc:648] HybridClock initialized: now 1781770104722468 us; error 46 us; skew 500 ppm
08:08:24.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.723410 30326 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:24.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:24 dist-test-slave-d25z krb5kdc[30144](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770104, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:24.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.730167 30326 init.cc:377] Logged in from keytab as kudu/127.29.109.124@KRBTEST.COM (short username kudu)
08:08:24.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.736444 30326 webserver.cc:533] Webserver started at http://127.29.109.124:38597/ using document root <none> and password file <none>
08:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.736519 30232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 7
08:08:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.737583 30326 fs_manager.cc:362] Metadata directory not provided
08:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.737848 30326 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.738471 30326 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:08:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.744378 30326 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data/instance:
08:08:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4eb4255300c14baca7ea451371aebb4b"
08:08:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:24 on dist-test-slave-d25z"
08:08:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:24.745560 30232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.745779 30326 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/wal/instance:
08:08:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4eb4255300c14baca7ea451371aebb4b"
08:08:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:24 on dist-test-slave-d25z"
08:08:24.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.754040 30326 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
08:08:24.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.760095 30352 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:24.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.761354 30326 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
08:08:24.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.761633 30326 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data,/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/wal
08:08:24.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4eb4255300c14baca7ea451371aebb4b"
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:24 on dist-test-slave-d25z"
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.761917 30326 fs_report.cc:389] FS layout report
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/wal
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/wal
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data/data
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:24.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:24.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.821139 30326 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:24.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.827198 30326 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:24.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.827606 30326 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:24.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.847998 30316 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } attempt: 7
08:08:24.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:24.860754 30316 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.109.124:36405: Network error: Client connection negotiation failed: client connection to 127.29.109.124:36405: connect: Connection refused (error 111)
08:08:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.931514 30326 rpc_server.cc:307] RPC server started. Bound to: 127.29.109.124:36405
08:08:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.931633 30405 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.109.124:36405 every 8 connection(s)
08:08:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.934288 30326 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data/info.pb
08:08:24.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.943339 30406 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:08:24.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.944264 30133 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu as pid 30326
08:08:24.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.944758 30133 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/wal/instance
08:08:24.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:24.965953 30406 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } has no permanent_uuid. Determining permanent_uuid...
08:08:24.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:24 dist-test-slave-d25z krb5kdc[30144](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770097, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.109.124@KRBTEST.COM
08:08:25.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:24 dist-test-slave-d25z krb5kdc[30144](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770104, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.124@KRBTEST.COM for kudu/127.29.109.126@KRBTEST.COM
08:08:25.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.014699 30406 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } has no permanent_uuid. Determining permanent_uuid...
08:08:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:25 dist-test-slave-d25z krb5kdc[30144](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770104, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.124@KRBTEST.COM for kudu/127.29.109.125@KRBTEST.COM
08:08:25.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.053566 30406 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } has no permanent_uuid. Determining permanent_uuid...
08:08:25.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:25 dist-test-slave-d25z krb5kdc[30144](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770097, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.109.125@KRBTEST.COM
08:08:25.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:25 dist-test-slave-d25z krb5kdc[30144](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770104, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.124@KRBTEST.COM for kudu/127.29.109.124@KRBTEST.COM
08:08:25.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:25 dist-test-slave-d25z krb5kdc[30144](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770097, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.109.126@KRBTEST.COM
08:08:25.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.093223 30406 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: Bootstrap starting.
08:08:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.105535 30406 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: Neither blocks nor log segments found. Creating new log.
08:08:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.107167 30133 external_mini_cluster.cc:955] 0 TS(s) registered with all masters
08:08:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.107472 30406 log.cc:826] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: Log is configured to *not* fsync() on all Append() calls
08:08:25.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.114643 30406 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: No bootstrap required, opened a new log
08:08:25.119 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:25.120 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
08:08:25.121 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
08:08:25.121 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
08:08:25.122 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.29.109.126"
port: 34513
}
bound_http_address {
host: "127.29.109.126"
port: 43499
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.29.109.125"
port: 38999
}
bound_http_address {
host: "127.29.109.125"
port: 33569
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.29.109.124"
port: 36405
}
bound_http_address {
host: "127.29.109.124"
port: 38597
}
}
}
08:08:25.123 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:08:25.125 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:25.125 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:08:25.125 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:08:25.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.145951 30406 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } }
08:08:25.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.146955 30406 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:08:25.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.147269 30406 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4eb4255300c14baca7ea451371aebb4b, State: Initialized, Role: FOLLOWER
08:08:25.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.148360 30406 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } }
08:08:25.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:25 dist-test-slave-d25z krb5kdc[30144](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770105, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:25.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:08:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.152417 30417 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } } }
08:08:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.153580 30417 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [sys.catalog]: This master's current role is: FOLLOWER
08:08:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.154795 30406 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [sys.catalog]: configured and running, proceeding with master startup.
08:08:25.156 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:25.158 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5cc
08:08:25.159 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:08:25.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:25.192198 30428 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:25.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:25.193825 30428 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:25.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.182971 30326 master_runner.cc:187] Error getting master registration for 127.29.109.126:34513: OK, instance_id { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" instance_seqno: 1781770099945637 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:08:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.208920 30326 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:08:25.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:25.334151 30358 tablet.cc:2406] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:08:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.444108 30417 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.444700 30417 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } }
08:08:25.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:25.448382 30417 leader_election.cc:290] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513), 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999)
08:08:26.161 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.109.126:34513
08:08:26.162 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:08:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:26.162860 30133 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu with pid 30161
08:08:26.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.192207 30355 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.29.109.126:34513 (error 108)
08:08:26.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.206161 30428 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:26.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.206485 30428 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:26.211 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:26.212 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.109.125:38999
08:08:26.212 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:08:26.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:26.212829 30133 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu with pid 30241
08:08:26.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.216964 30355 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513): Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111)
08:08:26.236 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:26.236 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.109.124:36405
08:08:26.236 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:08:26.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:26.237212 30133 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu with pid 30326
08:08:26.262 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:26.262 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.29.109.126:34513
08:08:26.262 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:08:26.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:26.263988 30133 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:26.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:26.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/logs
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data/info.pb
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.109.126:34513
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.109.126
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43499
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.109.126
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.109.84:43249
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34573/jwks.json
08:08:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:26.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.636343 30433 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:26.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.637179 30433 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:26.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.637953 30433 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:26.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.676017 30433 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:26.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.676347 30433 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:26.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.676777 30433 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.677103 30433 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.677429 30433 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:26.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.677739 30433 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:26.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.677992 30433 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34573/jwks.json
08:08:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:26.720914 30433 master_runner.cc:387] Master server non-default flags:
08:08:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.109.84:43249
08:08:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data
08:08:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal
08:08:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405
08:08:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab
08:08:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.109.126
08:08:26.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.109.126:34513
08:08:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34573/jwks.json
08:08:26.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data/info.pb
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.109.126
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43499
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/logs
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 07:43:20 UTC on e7f111948823
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12451
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:26.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:26.722812 30433 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:26.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:26.724231 30433 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:26.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.735855 30439 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.742892 30442 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:28.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:26.738238 30440 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:28.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.041371 30441 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1298 milliseconds
08:08:28.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.041494 30433 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.042733 30433 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:28.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.045938 30433 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:28.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.047374 30433 hybrid_clock.cc:648] HybridClock initialized: now 1781770108047313 us; error 75 us; skew 500 ppm
08:08:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.048270 30433 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:28 dist-test-slave-d25z krb5kdc[30144](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770108, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:28.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.055531 30433 init.cc:377] Logged in from keytab as kudu/127.29.109.126@KRBTEST.COM (short username kudu)
08:08:28.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.061765 30433 webserver.cc:533] Webserver started at http://127.29.109.126:43499/ using document root <none> and password file <none>
08:08:28.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.062644 30433 fs_manager.cc:362] Metadata directory not provided
08:08:28.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.062855 30433 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:28.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.071652 30433 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
08:08:28.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.076324 30451 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:28.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.077682 30433 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
08:08:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.078008 30433 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data,/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal
08:08:28.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2efcd7c2409142d18d168ab8cfbc2d72"
08:08:28.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:19 on dist-test-slave-d25z"
08:08:28.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.079917 30433 fs_report.cc:389] FS layout report
08:08:28.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data/data
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:28.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.136085 30433 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:28.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.141564 30433 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:28.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.142107 30433 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:28.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.223886 30433 rpc_server.cc:307] RPC server started. Bound to: 127.29.109.126:34513
08:08:28.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.224015 30502 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.109.126:34513 every 8 connection(s)
08:08:28.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.226861 30433 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/data/info.pb
08:08:28.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.227864 30133 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu as pid 30433
08:08:28.228 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:28.228 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.29.109.125:38999
08:08:28.229 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.231048 30133 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/logs
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data/info.pb
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.109.125:38999
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.109.125
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33569
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.109.125
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.109.84:43249
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34573/jwks.json
08:08:28.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:28.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:28.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.240599 30503 sys_catalog.cc:263] Verifying existing consensus state
08:08:28.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260618 08:08:28.241561 30503 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:08:28.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.242062 30433 master.cc:562] Master@127.29.109.126:34513 shutting down...
08:08:28.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.310828 30433 master.cc:584] Master@127.29.109.126:34513 shutdown complete.
08:08:28.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.315267 30433 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
08:08:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:08:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.581745 30505 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.582295 30505 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:28.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.582890 30505 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:28.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.618532 30505 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:28.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.618759 30505 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.619033 30505 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.619235 30505 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.619436 30505 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.619632 30505 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.619779 30505 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34573/jwks.json
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.659657 30505 master_runner.cc:387] Master server non-default flags:
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.109.84:43249
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.109.125
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.109.125:38999
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34573/jwks.json
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data/info.pb
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.109.125
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33569
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/logs
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:28.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:28.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:28.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:28.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:28.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:08:28.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:28.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 07:43:20 UTC on e7f111948823
08:08:28.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12451
08:08:28.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:28.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.660971 30505 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:28.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:28.661965 30505 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:28.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.674300 30511 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:30.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.678689 30514 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:30.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:28.675515 30512 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:30.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.024085 30513 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1345 milliseconds
08:08:30.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.024211 30505 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:30.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.025704 30505 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.028825 30505 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:30.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.030282 30505 hybrid_clock.cc:648] HybridClock initialized: now 1781770110030212 us; error 70 us; skew 500 ppm
08:08:30.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.031245 30505 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:30.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:30 dist-test-slave-d25z krb5kdc[30144](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770110, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:30.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.037242 30505 init.cc:377] Logged in from keytab as kudu/127.29.109.125@KRBTEST.COM (short username kudu)
08:08:30.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.042464 30505 webserver.cc:533] Webserver started at http://127.29.109.125:33569/ using document root <none> and password file <none>
08:08:30.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.043323 30505 fs_manager.cc:362] Metadata directory not provided
08:08:30.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.043525 30505 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.051971 30505 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.002s sys 0.004s
08:08:30.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.056399 30524 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.057603 30505 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.003s
08:08:30.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.057896 30505 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data,/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal
08:08:30.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5cf4af86dd6f4f69aea3307291ff88e6"
08:08:30.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:22 on dist-test-slave-d25z"
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.059723 30505 fs_report.cc:389] FS layout report
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data/data
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:30.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:30.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.136545 30505 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:30.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.145572 30505 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:30.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.146257 30505 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.232251 30505 rpc_server.cc:307] RPC server started. Bound to: 127.29.109.125:38999
08:08:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.232362 30575 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.109.125:38999 every 8 connection(s)
08:08:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.235307 30505 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/data/info.pb
08:08:30.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.243070 30133 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu as pid 30505
08:08:30.243 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:30.243 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.29.109.124:36405
08:08:30.243 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:08:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.245343 30133 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu
08:08:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/wal
08:08:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data
08:08:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:08:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:08:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:08:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/logs
08:08:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data/info.pb
08:08:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.109.124:36405
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.109.124
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38597
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.109.124
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.109.84:43249
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34573/jwks.json
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:08:30.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:08:30.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.247149 30576 sys_catalog.cc:263] Verifying existing consensus state
08:08:30.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260618 08:08:30.247992 30576 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:08:30.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.248459 30505 master.cc:562] Master@127.29.109.125:38999 shutting down...
08:08:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.321147 30505 master.cc:584] Master@127.29.109.125:38999 shutdown complete.
08:08:30.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.324810 30505 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
08:08:30.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:08:30.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.597904 30578 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:08:30.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.598538 30578 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:08:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.599145 30578 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:08:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.634878 30578 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:08:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.635134 30578 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:08:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.635421 30578 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:08:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.635668 30578 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:08:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.635882 30578 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:08:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.636085 30578 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:08:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.636245 30578 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34573/jwks.json
08:08:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.681420 30578 master_runner.cc:387] Master server non-default flags:
08:08:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:08:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.109.84:43249
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/wal
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/kudu.keytab
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.109.124
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.109.124:36405
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34573/jwks.json
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data/info.pb
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.109.124
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38597
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/logs
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision faec34f53d4c9dd1e3a6dd2b7314d2fc7aadff6c
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Jun 2026 07:43:20 UTC on e7f111948823
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12451
08:08:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:08:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.682914 30578 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:08:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:30.684139 30578 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:08:30.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.696334 30585 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:32.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:30.699193 30586 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:32.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:32.102764 30584 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 30578
08:08:32.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:32.351776 30578 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.654s user 0.570s sys 1.019s
08:08:32.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:32.352151 30578 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.655s user 0.570s sys 1.019s
08:08:32.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:32.354754 30588 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:08:32.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:32.357371 30587 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1652 milliseconds
08:08:32.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.357363 30578 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:08:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.358901 30578 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:08:32.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.361637 30578 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:08:32.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.363011 30578 hybrid_clock.cc:648] HybridClock initialized: now 1781770112362976 us; error 39 us; skew 500 ppm
08:08:32.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.363988 30578 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:08:32.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:32 dist-test-slave-d25z krb5kdc[30144](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770112, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:32.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.370838 30578 init.cc:377] Logged in from keytab as kudu/127.29.109.124@KRBTEST.COM (short username kudu)
08:08:32.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.375945 30578 webserver.cc:533] Webserver started at http://127.29.109.124:38597/ using document root <none> and password file <none>
08:08:32.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.376924 30578 fs_manager.cc:362] Metadata directory not provided
08:08:32.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.377126 30578 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:08:32.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.385598 30578 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
08:08:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.390269 30597 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:08:32.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.391453 30578 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.391832 30578 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data,/tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/wal
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4eb4255300c14baca7ea451371aebb4b"
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-18 08:08:24 on dist-test-slave-d25z"
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.394169 30578 fs_report.cc:389] FS layout report
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/wal
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/wal
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data/data
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:08:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:08:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:08:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:08:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:08:32.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.453292 30578 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:08:32.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.459172 30578 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:08:32.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.459702 30578 kserver.cc:163] Server-wide thread pool size limit: 3276
08:08:32.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.558647 30578 rpc_server.cc:307] RPC server started. Bound to: 127.29.109.124:36405
08:08:32.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.558759 30648 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.109.124:36405 every 8 connection(s)
08:08:32.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.561942 30578 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/master-2/data/info.pb
08:08:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.572587 30133 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu as pid 30578
08:08:32.573 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:08:32.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.573987 30649 sys_catalog.cc:263] Verifying existing consensus state
08:08:32.574 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:32.575 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:32.576 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.109.124:36405(127.29.109.124:36405)] connecting to peer
08:08:32.578 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.579 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.29.109.124:36405(127.29.109.124:36405)] Successfully connected to peer
08:08:32.579 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.579 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.580 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.583103 30649 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: Bootstrap starting.
08:08:32.594 [DEBUG - kudu-nio-2] (SecurityContext.java:222) Refreshing Kerberos credentials...
08:08:32.594 [DEBUG - kudu-nio-2] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskbTrDWn/test-tmp/mini-kudu-cluster7831306614193592052/krb5kdc/krb5cc
08:08:32.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:32 dist-test-slave-d25z krb5kdc[30144](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781770105, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:32.599 [DEBUG - kudu-nio-2] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:08:32.599 [INFO - kudu-nio-2] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
08:08:32.600 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.29.109.124
08:08:32.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:32 dist-test-slave-d25z krb5kdc[30144](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781770105, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.109.124@KRBTEST.COM
08:08:32.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.633306 30649 log.cc:826] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: Log is configured to *not* fsync() on all Append() calls
08:08:32.634 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xb31c4b9d, L:/127.0.0.1:55432 - R:/127.29.109.124:36405] using SASL/GSSAPI
08:08:32.641 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.124:36405: Catalog manager is not initialized. State: Starting
08:08:32.641 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.641675 30649 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:08:32.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.642820 30649 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: Bootstrap complete.
08:08:32.655 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:32.656 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:32.657 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.657 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.657 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.658 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.658 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.658 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.660 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.124:36405: Catalog manager is not initialized. State: Starting
08:08:32.660 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:32.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.667042 30649 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } }
08:08:32.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.669587 30649 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:08:32.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.669947 30649 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4eb4255300c14baca7ea451371aebb4b, State: Initialized, Role: FOLLOWER
08:08:32.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.670853 30649 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } }
08:08:32.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.673966 30659 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } } }
08:08:32.674 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:32.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.674911 30659 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [sys.catalog]: This master's current role is: FOLLOWER
08:08:32.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.675840 30649 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [sys.catalog]: configured and running, proceeding with master startup.
08:08:32.676 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:32.678 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.678 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.679 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.679086 30659 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.679626 30659 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } }
08:08:32.680 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.124:36405: Catalog manager is not initialized. State: Starting
08:08:32.681 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.681 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.681 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.681 [WARN - kudu-nio-5] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:32.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.683851 30659 leader_election.cc:290] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513), 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999)
08:08:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:32.688006 30600 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111)
08:08:32.695 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:32.697 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:32.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.700 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.700 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.700 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:32.704629 30599 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999): Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:32.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:32.705375 30600 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513): Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111)
08:08:32.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.705767 30600 leader_election.cc:304] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4eb4255300c14baca7ea451371aebb4b; no voters: 2efcd7c2409142d18d168ab8cfbc2d72, 5cf4af86dd6f4f69aea3307291ff88e6
08:08:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.706563 30659 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:08:32.707 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.124:36405: Catalog manager is not initialized. State: Starting
08:08:32.709 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:32.715754 30603 tablet.cc:2406] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:08:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:32.725561 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:32.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:32.725907 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:32.735 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:32.735 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:32.736 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.736 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.736 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.737 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.737 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.737 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.741 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:32.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.721014 30578 master_runner.cc:187] Error getting master registration for 127.29.109.126:34513: Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111),
08:08:32.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:32.745505 30578 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:08:32.754 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:32.755 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:32.756 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.756 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.756 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.756 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.756 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.756 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.759 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:32.835 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:32.835 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:32.836 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.836 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.837 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.837 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.837 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.837 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.840 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:32.954 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:32.955 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:32.955 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.955 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.956 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:32.956 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.956 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.957 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:32.959 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:33.155 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:33.156 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:33.156 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:33.156 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:33.156 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:33.157 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:33.157 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:33.157 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:33.160 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:33.615 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:33.615 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:33.616 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:33.616 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:33.616 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:33.616 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:33.616 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:33.617 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:33.619 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:33.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:33.729117 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:33.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:33.729385 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:34.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:34.216094 30681 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:34.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:34.216565 30681 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } }
08:08:34.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:34.218235 30681 leader_election.cc:290] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513), 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999)
08:08:34.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:34.226014 30599 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999): Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:34.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:34.226686 30600 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513): Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111)
08:08:34.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:34.227038 30600 leader_election.cc:304] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4eb4255300c14baca7ea451371aebb4b; no voters: 2efcd7c2409142d18d168ab8cfbc2d72, 5cf4af86dd6f4f69aea3307291ff88e6
08:08:34.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:34.227736 30681 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:08:34.475 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:34.476 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:34.476 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:34.477 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:34.477 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:34.477 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:34.479 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:34.479 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:34.481 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:34.732347 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:34.732676 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:35.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:35.735886 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:35.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:35.736191 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:35.875 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:35.876 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:35.876 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:35.876 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:35.877 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:35.877 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:35.877 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:35.877 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:35.880 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:36.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:36.562202 30686 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:36.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:36.562606 30686 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } }
08:08:36.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:36.564694 30686 leader_election.cc:290] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513), 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999)
08:08:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:36.571874 30600 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513): Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111)
08:08:36.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:36.572427 30599 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999): Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:36.572844 30599 leader_election.cc:304] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4eb4255300c14baca7ea451371aebb4b; no voters: 2efcd7c2409142d18d168ab8cfbc2d72, 5cf4af86dd6f4f69aea3307291ff88e6
08:08:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:36.573470 30686 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:08:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:36.739255 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:36.739591 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:37.234 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:37.235 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:37.236 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:37.236 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:37.237 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:37.237 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:37.237 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:37.238 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:37.239 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:37.536926 29838 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad2a0 after lost signal to thread 29711
08:08:37.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:37.538182 29838 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad240 after lost signal to thread 29841
08:08:37.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:37.742655 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:37.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:37.742954 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:37.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:37.749395 30578 master_runner.cc:187] Error getting master registration for 127.29.109.126:34513: Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111),
08:08:37.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:37.749850 30578 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:08:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:38.745826 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:38.746104 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:39.749505 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:39.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:39.749830 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:40.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:40.007531 30694 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:40.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:40.008029 30694 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } }
08:08:40.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:40.009979 30694 leader_election.cc:290] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513), 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999)
08:08:40.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:40.012274 30600 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111) [suppressed 5 similar messages]
08:08:40.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:40.015578 30599 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999): Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:40.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:40.016111 30600 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513): Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111)
08:08:40.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:40.016482 30600 leader_election.cc:304] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4eb4255300c14baca7ea451371aebb4b; no voters: 2efcd7c2409142d18d168ab8cfbc2d72, 5cf4af86dd6f4f69aea3307291ff88e6
08:08:40.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:40.017153 30694 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:08:40.574 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:40.575 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:40.576 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:40.576 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:40.576 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:40.577 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:40.577 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:40.577 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:40.592 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:40.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:40.753948 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:40.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:40.754401 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:41.074 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:41.075 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:41.076 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:41.076 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:41.076 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:41.076 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:41.076 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:41.076 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:41.079 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:41.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:41.757650 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:41.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:41.757932 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:42.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:42 dist-test-slave-d25z krb5kdc[30144](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770122, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:42.375855 30593 init.cc:295] Successfully reacquired a new kerberos TGT
08:08:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:42.754112 30578 master_runner.cc:187] Error getting master registration for 127.29.109.126:34513: Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111),
08:08:42.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:42.754514 30578 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:08:42.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:42.761025 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:42.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:42.761312 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:43.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:43.764546 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:43.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:43.764823 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:43.835 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:43.835 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:43.836 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:43.836 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:43.836 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:43.837 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:43.837 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:43.837 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:43.839 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:44.767793 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:44.768077 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:45.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:45.771230 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:45.771526 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:45.774 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:45.775 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:45.775 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:45.775 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:45.775 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:45.777 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:45.777 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:45.777 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:45.779 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:46.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:46.774586 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:46.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:46.774873 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:47.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:47.150203 30708 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:47.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:47.150667 30708 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } }
08:08:47.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:47.152523 30708 leader_election.cc:290] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513), 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999)
08:08:47.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:47.154946 30600 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111) [suppressed 1 similar messages]
08:08:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:47.158681 30599 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999): Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:47.159451 30600 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513): Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111)
08:08:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:47.159883 30600 leader_election.cc:304] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4eb4255300c14baca7ea451371aebb4b; no voters: 2efcd7c2409142d18d168ab8cfbc2d72, 5cf4af86dd6f4f69aea3307291ff88e6
08:08:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:47.160597 30708 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:08:47.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:47.719411 29838 debug-util.cc:398] Leaking SignalData structure 0x7b08000adb80 after lost signal to thread 29711
08:08:47.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:47.720701 29838 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba040 after lost signal to thread 29841
08:08:47.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:47.757833 30578 master_runner.cc:187] Error getting master registration for 127.29.109.126:34513: Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111),
08:08:47.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:47.758268 30578 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:08:47.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:47.778131 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:47.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:47.778478 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:48.115 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:48.115 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:48.116 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:48.116 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:48.116 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:48.116 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:48.117 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:48.117 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:48.119 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:48.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:48.781641 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:48.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:48.781971 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:49.785265 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:49.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:49.785624 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:50.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:50.789696 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:50.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:50.790129 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:51.695 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:51.696 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:51.697 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:51.697 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:51.697 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:51.697 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:51.697 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:51.697 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:51.701 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:51.793289 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:51.793582 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 18 08:08:52 dist-test-slave-d25z krb5kdc[30144](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781770132, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.109.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:08:52.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:52.379105 30593 init.cc:295] Successfully reacquired a new kerberos TGT
08:08:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:52.762802 30578 master_runner.cc:187] Error getting master registration for 127.29.109.126:34513: Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111),
08:08:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:52.763362 30578 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:08:52.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:52.796880 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:52.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:52.797192 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:53.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:53.800279 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:53.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:53.800648 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:54.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:54.803656 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:54.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:54.803960 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:55.214 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:55.215 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:55.216 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:55.216 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:55.216 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:55.217 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:55.217 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:55.217 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:55.219 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:55.806866 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:55.807123 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:56.176098 30726 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:56.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:56.177078 30726 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } }
08:08:56.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:56.179203 30726 leader_election.cc:290] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513), 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999)
08:08:56.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:56.181511 30600 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111) [suppressed 1 similar messages]
08:08:56.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:56.184770 30600 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513): Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111)
08:08:56.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:56.185446 30599 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999): Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:56.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:56.185815 30599 leader_election.cc:304] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4eb4255300c14baca7ea451371aebb4b; no voters: 2efcd7c2409142d18d168ab8cfbc2d72, 5cf4af86dd6f4f69aea3307291ff88e6
08:08:56.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:56.186478 30726 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:08:56.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:56.810593 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:56.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:56.810914 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:57.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:57.766636 30578 master_runner.cc:187] Error getting master registration for 127.29.109.126:34513: Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111),
08:08:57.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:57.767117 30578 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:08:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:57.814098 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:57.814442 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:58.614 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:08:58.615 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:08:58.615 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:58.616 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:58.616 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:08:58.617 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:58.617 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:58.618 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:08:58.620 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:08:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:58.819060 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:58.819753 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:59.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:59.610728 30735 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:08:59.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:59.611157 30735 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2efcd7c2409142d18d168ab8cfbc2d72" member_type: VOTER last_known_addr { host: "127.29.109.126" port: 34513 } } peers { permanent_uuid: "5cf4af86dd6f4f69aea3307291ff88e6" member_type: VOTER last_known_addr { host: "127.29.109.125" port: 38999 } } peers { permanent_uuid: "4eb4255300c14baca7ea451371aebb4b" member_type: VOTER last_known_addr { host: "127.29.109.124" port: 36405 } }
08:08:59.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:59.613015 30735 leader_election.cc:290] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513), 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999)
08:08:59.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:59.618551 30600 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2efcd7c2409142d18d168ab8cfbc2d72 (127.29.109.126:34513): Network error: Client connection negotiation failed: client connection to 127.29.109.126:34513: connect: Connection refused (error 111)
08:08:59.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:59.619091 30599 leader_election.cc:336] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5cf4af86dd6f4f69aea3307291ff88e6 (127.29.109.125:38999): Network error: Client connection negotiation failed: client connection to 127.29.109.125:38999: connect: Connection refused (error 111)
08:08:59.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:59.619499 30599 leader_election.cc:304] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4eb4255300c14baca7ea451371aebb4b; no voters: 2efcd7c2409142d18d168ab8cfbc2d72, 5cf4af86dd6f4f69aea3307291ff88e6
08:08:59.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:08:59.620183 30735 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:08:59.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:59.823004 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:08:59.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:59.823284 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:08:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:59.846858 29560 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7580 after lost signal to thread 29495
08:08:59.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:08:59.847631 29560 debug-util.cc:398] Leaking SignalData structure 0x7b0800097120 after lost signal to thread 29563
08:09:00.375 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:09:00.379 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:09:00.379 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:09:00.379 [INFO - pool-44-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:09:00.379 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:09:00.381 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:09:00.381 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:09:00.381 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:09:00.384 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:09:00.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:09:00.829160 30677 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 4eb4255300c14baca7ea451371aebb4b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:09:00.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260618 08:09:00.829723 30677 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:09:00.994 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:09:00.995 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:09:00.996 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:09:00.996 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:09:00.996 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:09:00.996 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:09:00.996 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:09:00.996 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:09:00.999 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:09:01.795 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.126:34513(127.29.109.126:34513)] connecting to peer
08:09:01.796 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.29.109.125:38999(127.29.109.125:38999)] connecting to peer
08:09:01.796 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:09:01.797 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.29.109.126:34513(127.29.109.126:34513)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:09:01.797 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:09:01.797 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.126:34513: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513
08:09:01.797 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.109.125:38999(127.29.109.125:38999)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:09:01.797 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.109.125:38999: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
08:09:01.801 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405; will retry
08:09:01.803 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29227), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [6ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [6ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [67ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [68ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [84ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [85ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [87ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [87ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [106ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [107ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [108ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [108ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [121ms] refreshing cache from master, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [126ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [127ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [134ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [137ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [163ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [164ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [168ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [168ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [183ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [183ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [186ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [186ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [261ms] refreshing cache from master, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [263ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [264ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [264ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [267ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [267ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [381ms] refreshing cache from master, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [383ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [383ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [384ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [386ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [386ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [581ms] refreshing cache from master, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [583ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [583ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [584ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [584ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [587ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [587ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1041ms] refreshing cache from master, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [1042ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [1043ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [1043ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1044ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1046ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [1046ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1901ms] refreshing cache from master, [1902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [1902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [1903ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [1904ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1906ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1908ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [1908ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [3301ms] refreshing cache from master, [3302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [3303ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [3303ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [3304ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [3304ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [3307ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [3307ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [4661ms] refreshing cache from master, [4661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [4662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [4663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [4664ms] too many traces: truncated at 100 traces, deferred=Deferred@1325738348(state=PENDING, result=null, callback=(continuation of Deferred@1088340043 after retry RPC after error@1986246590) -> (continuation of Deferred@1986586902 after retry RPC after error@272691454) -> (continuation of Deferred@648717861 after retry RPC after error@1302276873) -> (continuation of Deferred@2081315829 after retry RPC after error@1801356170) -> (continuation of Deferred@406231882 after retry RPC after error@742730790) -> (continuation of Deferred@572132264 after retry RPC after error@1036488370) -> (continuation of Deferred@285716107 after retry RPC after error@338072721) -> (continuation of Deferred@1712560886 after retry RPC after error@1738554008) -> (continuation of Deferred@1275048721 after retry RPC after error@1801266403) -> (continuation of Deferred@291751764 after retry RPC after error@2003917594) -> (continuation of Deferred@1101011422 after retry RPC after error@1129862965) -> (continuation of Deferred@1284403565 after retry RPC after error@828004973) -> (continuation of Deferred@1886772081 after retry RPC after error@1905642421) -> (continuation of Deferred@1320147443 after retry RPC after error@767912825) -> (continuation of Deferred@1941275619 after retry RPC after error@748824131) -> (continuation of Deferred@485626469 after retry RPC after error@451558433) -> (continuation of Deferred@1168051716 after retry RPC after error@2024978634) -> (continuation of Deferred@1960832131 after retry RPC after error@1067441421) -> (continuation of Deferred@38607828 after retry RPC after error@1876199359) -> (continuation of Deferred@1692718386 after retry RPC after error@626137878) -> (continuation of Deferred@1336616915 after retry RPC after error@67741926) -> (continuation of Deferred@1174843408 after retry RPC after error@1941539181) -> (continuation of Deferred@1706737014 after retry RPC after error@742221759), errback=(continuation of Deferred@1088340043 after retry RPC after error@1986246590) -> (continuation of Deferred@1986586902 after retry RPC after error@272691454) -> (continuation of Deferred@648717861 after retry RPC after error@1302276873) -> (continuation of Deferred@2081315829 after retry RPC after error@1801356170) -> (continuation of Deferred@406231882 after retry RPC after error@742730790) -> (continuation of Deferred@572132264 after retry RPC after error@1036488370) -> (continuation of Deferred@285716107 after retry RPC after error@338072721) -> (continuation of Deferred@1712560886 after retry RPC after error@1738554008) -> (continuation of Deferred@1275048721 after retry RPC after error@1801266403) -> (continuation of Deferred@291751764 after retry RPC after error@2003917594) -> (continuation of Deferred@1101011422 after retry RPC after error@1129862965) -> (continuation of Deferred@1284403565 after retry RPC after error@828004973) -> (continuation of Deferred@1886772081 after retry RPC after error@1905642421) -> (continuation of Deferred@1320147443 after retry RPC after error@767912825) -> (continuation of Deferred@1941275619 after retry RPC after error@748824131) -> (continuation of Deferred@485626469 after retry RPC after error@451558433) -> (continuation of Deferred@1168051716 after retry RPC after error@2024978634) -> (continuation of Deferred@1960832131 after retry RPC after error@1067441421) -> (continuation of Deferred@38607828 after retry RPC after error@1876199359) -> (continuation of Deferred@1692718386 after retry RPC after error@626137878) -> (continuation of Deferred@1336616915 after retry RPC after error@67741926) -> (continuation of Deferred@1174843408 after retry RPC after error@1941539181) -> (continuation of Deferred@1706737014 after retry RPC after error@742221759)))
08:09:01.805 [DEBUG - main] (Connection.java:575) [peer master-127.29.109.126:34513(127.29.109.126:34513)] disconnecting while in state TERMINATED
08:09:01.805 [DEBUG - main] (Connection.java:575) [peer master-127.29.109.125:38999(127.29.109.125:38999)] disconnecting while in state TERMINATED
08:09:01.806 [DEBUG - main] (Connection.java:575) [peer master-127.29.109.124:36405(127.29.109.124:36405)] disconnecting while in state READY
08:09:01.806 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:09:01.808 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.29.109.124:36405(127.29.109.124:36405)] handling channelInactive
08:09:01.809 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.29.109.124:36405(127.29.109.124:36405)] cleaning up while in state READY due to: connection closed
08:09:01.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:09:01.809882 30133 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu with pid 30578
08:09:03.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-18T08:09:03Z chronyd exiting
08:09:04.062 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:09:04.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:09:04.066298 29409 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu with pid 29691
08:09:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:09:04.094669 29409 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu with pid 29852
08:09:04.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:09:04.125008 29409 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu with pid 29988
08:09:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:09:04.156291 29409 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu with pid 29421
08:09:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:09:04.193229 29409 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu with pid 29491
08:09:04.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260618 08:09:04.231274 29409 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskbTrDWn/build/tsan/bin/kudu with pid 29566
08:09:05.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-18T08:09:05Z chronyd exiting
08:09:06.434 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29227), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [6ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [6ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [67ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [68ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [84ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [85ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [87ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [87ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [106ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [107ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [108ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [108ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [121ms] refreshing cache from master, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [126ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [127ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [134ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [137ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [163ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [164ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [168ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [168ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [183ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [183ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [186ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [186ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [261ms] refreshing cache from master, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [263ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [264ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [264ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [267ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [267ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [381ms] refreshing cache from master, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [383ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [383ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [384ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [386ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [386ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [581ms] refreshing cache from master, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [583ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [583ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [584ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [584ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [587ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [587ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1041ms] refreshing cache from master, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [1042ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [1043ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [1043ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1044ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1046ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [1046ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1901ms] refreshing cache from master, [1902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [1902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [1903ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [1904ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1906ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1908ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [1908ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [3301ms] refreshing cache from master, [3302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [3303ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [3303ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [3304ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [3304ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [3307ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [3307ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [4661ms] refreshing cache from master, [4661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [4662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [4663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [4664ms] too many traces: truncated at 100 traces, deferred=Deferred@1325738348(state=PENDING, result=null, callback=(continuation of Deferred@1088340043 after retry RPC after error@1986246590) -> (continuation of Deferred@1986586902 after retry RPC after error@272691454) -> (continuation of Deferred@648717861 after retry RPC after error@1302276873) -> (continuation of Deferred@2081315829 after retry RPC after error@1801356170) -> (continuation of Deferred@406231882 after retry RPC after error@742730790) -> (continuation of Deferred@572132264 after retry RPC after error@1036488370) -> (continuation of Deferred@285716107 after retry RPC after error@338072721) -> (continuation of Deferred@1712560886 after retry RPC after error@1738554008) -> (continuation of Deferred@1275048721 after retry RPC after error@1801266403) -> (continuation of Deferred@291751764 after retry RPC after error@2003917594) -> (continuation of Deferred@1101011422 after retry RPC after error@1129862965) -> (continuation of Deferred@1284403565 after retry RPC after error@828004973) -> (continuation of Deferred@1886772081 after retry RPC after error@1905642421) -> (continuation of Deferred@1320147443 after retry RPC after error@767912825) -> (continuation of Deferred@1941275619 after retry RPC after error@748824131) -> (continuation of Deferred@485626469 after retry RPC after error@451558433) -> (continuation of Deferred@1168051716 after retry RPC after error@2024978634) -> (continuation of Deferred@1960832131 after retry RPC after error@1067441421) -> (continuation of Deferred@38607828 after retry RPC after error@1876199359) -> (continuation of Deferred@1692718386 after retry RPC after error@626137878) -> (continuation of Deferred@1336616915 after retry RPC after error@67741926) -> (continuation of Deferred@1174843408 after retry RPC after error@1941539181) -> (continuation of Deferred@1706737014 after retry RPC after error@742221759), errback=(continuation of Deferred@1088340043 after retry RPC after error@1986246590) -> (continuation of Deferred@1986586902 after retry RPC after error@272691454) -> (continuation of Deferred@648717861 after retry RPC after error@1302276873) -> (continuation of Deferred@2081315829 after retry RPC after error@1801356170) -> (continuation of Deferred@406231882 after retry RPC after error@742730790) -> (continuation of Deferred@572132264 after retry RPC after error@1036488370) -> (continuation of Deferred@285716107 after retry RPC after error@338072721) -> (continuation of Deferred@1712560886 after retry RPC after error@1738554008) -> (continuation of Deferred@1275048721 after retry RPC after error@1801266403) -> (continuation of Deferred@291751764 after retry RPC after error@2003917594) -> (continuation of Deferred@1101011422 after retry RPC after error@1129862965) -> (continuation of Deferred@1284403565 after retry RPC after error@828004973) -> (continuation of Deferred@1886772081 after retry RPC after error@1905642421) -> (continuation of Deferred@1320147443 after retry RPC after error@767912825) -> (continuation of Deferred@1941275619 after retry RPC after error@748824131) -> (continuation of Deferred@485626469 after retry RPC after error@451558433) -> (continuation of Deferred@1168051716 after retry RPC after error@2024978634) -> (continuation of Deferred@1960832131 after retry RPC after error@1067441421) -> (continuation of Deferred@38607828 after retry RPC after error@1876199359) -> (continuation of Deferred@1692718386 after retry RPC after error@626137878) -> (continuation of Deferred@1336616915 after retry RPC after error@67741926) -> (continuation of Deferred@1174843408 after retry RPC after error@1941539181) -> (continuation of Deferred@1706737014 after retry RPC after error@742221759)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.29.109.125:38999
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29227), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [6ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [6ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [67ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [68ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [84ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [85ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [87ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [87ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [106ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [107ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [108ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [108ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [121ms] refreshing cache from master, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [126ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [127ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [134ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [137ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [163ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [164ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [168ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [168ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [183ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [183ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [186ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [186ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [261ms] refreshing cache from master, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [263ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [264ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [264ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [267ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [267ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [381ms] refreshing cache from master, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [383ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [383ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [384ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [386ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [386ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [581ms] refreshing cache from master, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [583ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [583ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [584ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [584ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [587ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [587ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1041ms] refreshing cache from master, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [1042ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [1043ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [1043ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1044ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1046ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [1046ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1901ms] refreshing cache from master, [1902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [1902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [1903ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [1904ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1906ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1908ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [1908ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [3301ms] refreshing cache from master, [3302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [3303ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [3303ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [3304ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [3304ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [3307ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [3307ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [4661ms] refreshing cache from master, [4661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [4662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [4663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [4664ms] too many traces: truncated at 100 traces, deferred=Deferred@1325738348(state=PENDING, result=null, callback=(continuation of Deferred@1088340043 after retry RPC after error@1986246590) -> (continuation of Deferred@1986586902 after retry RPC after error@272691454) -> (continuation of Deferred@648717861 after retry RPC after error@1302276873) -> (continuation of Deferred@2081315829 after retry RPC after error@1801356170) -> (continuation of Deferred@406231882 after retry RPC after error@742730790) -> (continuation of Deferred@572132264 after retry RPC after error@1036488370) -> (continuation of Deferred@285716107 after retry RPC after error@338072721) -> (continuation of Deferred@1712560886 after retry RPC after error@1738554008) -> (continuation of Deferred@1275048721 after retry RPC after error@1801266403) -> (continuation of Deferred@291751764 after retry RPC after error@2003917594) -> (continuation of Deferred@1101011422 after retry RPC after error@1129862965) -> (continuation of Deferred@1284403565 after retry RPC after error@828004973) -> (continuation of Deferred@1886772081 after retry RPC after error@1905642421) -> (continuation of Deferred@1320147443 after retry RPC after error@767912825) -> (continuation of Deferred@1941275619 after retry RPC after error@748824131) -> (continuation of Deferred@485626469 after retry RPC after error@451558433) -> (continuation of Deferred@1168051716 after retry RPC after error@2024978634) -> (continuation of Deferred@1960832131 after retry RPC after error@1067441421) -> (continuation of Deferred@38607828 after retry RPC after error@1876199359) -> (continuation of Deferred@1692718386 after retry RPC after error@626137878) -> (continuation of Deferred@1336616915 after retry RPC after error@67741926) -> (continuation of Deferred@1174843408 after retry RPC after error@1941539181) -> (continuation of Deferred@1706737014 after retry RPC after error@742221759), errback=(continuation of Deferred@1088340043 after retry RPC after error@1986246590) -> (continuation of Deferred@1986586902 after retry RPC after error@272691454) -> (continuation of Deferred@648717861 after retry RPC after error@1302276873) -> (continuation of Deferred@2081315829 after retry RPC after error@1801356170) -> (continuation of Deferred@406231882 after retry RPC after error@742730790) -> (continuation of Deferred@572132264 after retry RPC after error@1036488370) -> (continuation of Deferred@285716107 after retry RPC after error@338072721) -> (continuation of Deferred@1712560886 after retry RPC after error@1738554008) -> (continuation of Deferred@1275048721 after retry RPC after error@1801266403) -> (continuation of Deferred@291751764 after retry RPC after error@2003917594) -> (continuation of Deferred@1101011422 after retry RPC after error@1129862965) -> (continuation of Deferred@1284403565 after retry RPC after error@828004973) -> (continuation of Deferred@1886772081 after retry RPC after error@1905642421) -> (continuation of Deferred@1320147443 after retry RPC after error@767912825) -> (continuation of Deferred@1941275619 after retry RPC after error@748824131) -> (continuation of Deferred@485626469 after retry RPC after error@451558433) -> (continuation of Deferred@1168051716 after retry RPC after error@2024978634) -> (continuation of Deferred@1960832131 after retry RPC after error@1067441421) -> (continuation of Deferred@38607828 after retry RPC after error@1876199359) -> (continuation of Deferred@1692718386 after retry RPC after error@626137878) -> (continuation of Deferred@1336616915 after retry RPC after error@67741926) -> (continuation of Deferred@1174843408 after retry RPC after error@1941539181) -> (continuation of Deferred@1706737014 after retry RPC after error@742221759)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.29.109.125:38999
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
08:09:06.607 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: giving up after 1 attempts
08:09:06.608 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29227), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [6ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [6ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [67ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [68ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [84ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [85ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [87ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [87ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [106ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [107ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [108ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [108ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [121ms] refreshing cache from master, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [126ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [127ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [134ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [137ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [163ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [164ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [168ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [168ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [183ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [183ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [186ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [186ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [261ms] refreshing cache from master, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [263ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [264ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [264ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [267ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [267ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [381ms] refreshing cache from master, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [383ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [383ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [384ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [386ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [386ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [581ms] refreshing cache from master, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [583ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [583ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [584ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [584ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [587ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [587ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1041ms] refreshing cache from master, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [1042ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [1043ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [1043ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1044ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1046ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [1046ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1901ms] refreshing cache from master, [1902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [1902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [1903ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [1904ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1906ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1908ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [1908ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [3301ms] refreshing cache from master, [3302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [3303ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [3303ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [3304ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [3304ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [3307ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [3307ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [4661ms] refreshing cache from master, [4661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [4662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [4663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [4664ms] too many traces: truncated at 100 traces, deferred=Deferred@1325738348(state=PENDING, result=null, callback=(continuation of Deferred@1088340043 after retry RPC after error@1986246590) -> (continuation of Deferred@1986586902 after retry RPC after error@272691454) -> (continuation of Deferred@648717861 after retry RPC after error@1302276873) -> (continuation of Deferred@2081315829 after retry RPC after error@1801356170) -> (continuation of Deferred@406231882 after retry RPC after error@742730790) -> (continuation of Deferred@572132264 after retry RPC after error@1036488370) -> (continuation of Deferred@285716107 after retry RPC after error@338072721) -> (continuation of Deferred@1712560886 after retry RPC after error@1738554008) -> (continuation of Deferred@1275048721 after retry RPC after error@1801266403) -> (continuation of Deferred@291751764 after retry RPC after error@2003917594) -> (continuation of Deferred@1101011422 after retry RPC after error@1129862965) -> (continuation of Deferred@1284403565 after retry RPC after error@828004973) -> (continuation of Deferred@1886772081 after retry RPC after error@1905642421) -> (continuation of Deferred@1320147443 after retry RPC after error@767912825) -> (continuation of Deferred@1941275619 after retry RPC after error@748824131) -> (continuation of Deferred@485626469 after retry RPC after error@451558433) -> (continuation of Deferred@1168051716 after retry RPC after error@2024978634) -> (continuation of Deferred@1960832131 after retry RPC after error@1067441421) -> (continuation of Deferred@38607828 after retry RPC after error@1876199359) -> (continuation of Deferred@1692718386 after retry RPC after error@626137878) -> (continuation of Deferred@1336616915 after retry RPC after error@67741926) -> (continuation of Deferred@1174843408 after retry RPC after error@1941539181) -> (continuation of Deferred@1706737014 after retry RPC after error@742221759), errback=(continuation of Deferred@1088340043 after retry RPC after error@1986246590) -> (continuation of Deferred@1986586902 after retry RPC after error@272691454) -> (continuation of Deferred@648717861 after retry RPC after error@1302276873) -> (continuation of Deferred@2081315829 after retry RPC after error@1801356170) -> (continuation of Deferred@406231882 after retry RPC after error@742730790) -> (continuation of Deferred@572132264 after retry RPC after error@1036488370) -> (continuation of Deferred@285716107 after retry RPC after error@338072721) -> (continuation of Deferred@1712560886 after retry RPC after error@1738554008) -> (continuation of Deferred@1275048721 after retry RPC after error@1801266403) -> (continuation of Deferred@291751764 after retry RPC after error@2003917594) -> (continuation of Deferred@1101011422 after retry RPC after error@1129862965) -> (continuation of Deferred@1284403565 after retry RPC after error@828004973) -> (continuation of Deferred@1886772081 after retry RPC after error@1905642421) -> (continuation of Deferred@1320147443 after retry RPC after error@767912825) -> (continuation of Deferred@1941275619 after retry RPC after error@748824131) -> (continuation of Deferred@485626469 after retry RPC after error@451558433) -> (continuation of Deferred@1168051716 after retry RPC after error@2024978634) -> (continuation of Deferred@1960832131 after retry RPC after error@1067441421) -> (continuation of Deferred@38607828 after retry RPC after error@1876199359) -> (continuation of Deferred@1692718386 after retry RPC after error@626137878) -> (continuation of Deferred@1336616915 after retry RPC after error@67741926) -> (continuation of Deferred@1174843408 after retry RPC after error@1941539181) -> (continuation of Deferred@1706737014 after retry RPC after error@742221759)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.29.109.125:38999
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29227), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [6ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [6ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [67ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [68ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [84ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [85ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [87ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [87ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [106ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [107ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [108ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [108ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [121ms] refreshing cache from master, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [126ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [127ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [134ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [137ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [163ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [164ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [168ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [168ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [183ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [183ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [186ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [186ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [261ms] refreshing cache from master, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [262ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [263ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [264ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [264ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [267ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [267ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [381ms] refreshing cache from master, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [382ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [383ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [383ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [384ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [386ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [386ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [581ms] refreshing cache from master, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [583ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [583ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [584ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [584ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [587ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [587ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1041ms] refreshing cache from master, [1041ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [1042ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [1043ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [1043ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1044ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1046ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [1046ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1901ms] refreshing cache from master, [1902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [1902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [1903ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [1904ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [1906ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [1908ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [1908ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [3301ms] refreshing cache from master, [3302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [3303ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [3303ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [3304ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.125:38999: Network error: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999, [3304ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.126:34513: Network error: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [3307ms] Sub RPC ConnectToMaster: received response from server master-127.29.109.124:36405: OK, [3307ms] delaying RPC due to: Service unavailable: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513, [4661ms] refreshing cache from master, [4661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.126:34513, [4662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.125:38999, [4663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.109.124:36405, [4664ms] too many traces: truncated at 100 traces, deferred=Deferred@1325738348(state=PENDING, result=null, callback=(continuation of Deferred@1088340043 after retry RPC after error@1986246590) -> (continuation of Deferred@1986586902 after retry RPC after error@272691454) -> (continuation of Deferred@648717861 after retry RPC after error@1302276873) -> (continuation of Deferred@2081315829 after retry RPC after error@1801356170) -> (continuation of Deferred@406231882 after retry RPC after error@742730790) -> (continuation of Deferred@572132264 after retry RPC after error@1036488370) -> (continuation of Deferred@285716107 after retry RPC after error@338072721) -> (continuation of Deferred@1712560886 after retry RPC after error@1738554008) -> (continuation of Deferred@1275048721 after retry RPC after error@1801266403) -> (continuation of Deferred@291751764 after retry RPC after error@2003917594) -> (continuation of Deferred@1101011422 after retry RPC after error@1129862965) -> (continuation of Deferred@1284403565 after retry RPC after error@828004973) -> (continuation of Deferred@1886772081 after retry RPC after error@1905642421) -> (continuation of Deferred@1320147443 after retry RPC after error@767912825) -> (continuation of Deferred@1941275619 after retry RPC after error@748824131) -> (continuation of Deferred@485626469 after retry RPC after error@451558433) -> (continuation of Deferred@1168051716 after retry RPC after error@2024978634) -> (continuation of Deferred@1960832131 after retry RPC after error@1067441421) -> (continuation of Deferred@38607828 after retry RPC after error@1876199359) -> (continuation of Deferred@1692718386 after retry RPC after error@626137878) -> (continuation of Deferred@1336616915 after retry RPC after error@67741926) -> (continuation of Deferred@1174843408 after retry RPC after error@1941539181) -> (continuation of Deferred@1706737014 after retry RPC after error@742221759), errback=(continuation of Deferred@1088340043 after retry RPC after error@1986246590) -> (continuation of Deferred@1986586902 after retry RPC after error@272691454) -> (continuation of Deferred@648717861 after retry RPC after error@1302276873) -> (continuation of Deferred@2081315829 after retry RPC after error@1801356170) -> (continuation of Deferred@406231882 after retry RPC after error@742730790) -> (continuation of Deferred@572132264 after retry RPC after error@1036488370) -> (continuation of Deferred@285716107 after retry RPC after error@338072721) -> (continuation of Deferred@1712560886 after retry RPC after error@1738554008) -> (continuation of Deferred@1275048721 after retry RPC after error@1801266403) -> (continuation of Deferred@291751764 after retry RPC after error@2003917594) -> (continuation of Deferred@1101011422 after retry RPC after error@1129862965) -> (continuation of Deferred@1284403565 after retry RPC after error@828004973) -> (continuation of Deferred@1886772081 after retry RPC after error@1905642421) -> (continuation of Deferred@1320147443 after retry RPC after error@767912825) -> (continuation of Deferred@1941275619 after retry RPC after error@748824131) -> (continuation of Deferred@485626469 after retry RPC after error@451558433) -> (continuation of Deferred@1168051716 after retry RPC after error@2024978634) -> (continuation of Deferred@1960832131 after retry RPC after error@1067441421) -> (continuation of Deferred@38607828 after retry RPC after error@1876199359) -> (continuation of Deferred@1692718386 after retry RPC after error@626137878) -> (continuation of Deferred@1336616915 after retry RPC after error@67741926) -> (continuation of Deferred@1174843408 after retry RPC after error@1941539181) -> (continuation of Deferred@1706737014 after retry RPC after error@742221759)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.29.109.126:34513,127.29.109.125:38999,127.29.109.124:36405) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.126:34513(127.29.109.126:34513): Connection refused: /127.29.109.126:34513,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.29.109.125:38999(127.29.109.125:38999): Connection refused: /127.29.109.125:38999
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.29.109.125:38999
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)