03:29:00.677 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:29:00.690 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskK6vj3Y/test-tmp
03:29:00.692 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:29:00.696 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskK6vj3Y/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:29:00.957 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:29:01.103 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:29:01.104 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:29:01.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:01.109205 4382 mini_oidc.cc:152] Starting JWKS server
03:29:01.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:01.113616 4382 webserver.cc:460] Webserver started at http://127.0.0.1:43105/ using document root <none> and password file <none>
03:29:01.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:01.114218 4382 mini_oidc.cc:189] Starting OIDC Discovery server
03:29:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:01.116389 4382 webserver.cc:460] Webserver started at http://127.0.0.1:45469/ using document root <none> and password file <none>
03:29:01.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T03:29:01Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:29:01.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T03:29:01Z Disabled control of system clock
03:29:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:01.166169 4382 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:01.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/wal
03:29:01.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/data
03:29:01.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:29:01.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:29:01.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:29:01.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:01.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:01.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:01.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/logs
03:29:01.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/data/info.pb
03:29:01.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:01.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:29:01.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:29:01.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:29:01.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:29:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:29:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:29:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.71.190:42417
03:29:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.71.190
03:29:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.71.148:42663
03:29:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:01.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:01.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:01.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.71.190:42417,127.4.71.189:37207,127.4.71.188:40861
03:29:01.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:01.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43105/jwks.json with env {}
03:29:01.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:01.497095 4394 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:29:01.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:01.497633 4394 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:29:01.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:01.498209 4394 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:29:01.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:01.529433 4394 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:29:01.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:01.529722 4394 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:29:01.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:01.529908 4394 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:29:01.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:01.530166 4394 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:29:01.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:01.530431 4394 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:29:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:01.530620 4394 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43105/jwks.json
03:29:01.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:01.567071 4394 master_runner.cc:386] Master server non-default flags:
03:29:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.71.148:42663
03:29:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/data
03:29:01.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/wal
03:29:01.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:01.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.71.190:42417,127.4.71.189:37207,127.4.71.188:40861
03:29:01.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:01.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:01.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:01.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.71.190:42417
03:29:01.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:01.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:29:01.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:01.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43105/jwks.json
03:29:01.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:01.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:01.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/data/info.pb
03:29:01.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.71.190
03:29:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:29:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:29:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:29:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/logs
03:29:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:29:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:29:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:29:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:29:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:29:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:29:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:29:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:29:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:29:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:01.568408 4394 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:29:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:01.570144 4394 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:29:01.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:01.590814 4400 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:01.591113 4403 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:01.594178 4394 server_base.cc:1034] running on GCE node
03:29:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:01.591636 4401 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:02.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.818830 4394 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:29:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.821882 4394 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:29:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.823359 4394 hybrid_clock.cc:648] HybridClock initialized: now 1790220542823332 us; error 37 us; skew 500 ppm
03:29:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.824206 4394 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:29:02.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.830657 4394 webserver.cc:460] Webserver started at http://127.4.71.190:36265/ using document root <none> and password file <none>
03:29:02.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.831630 4394 fs_manager.cc:359] Metadata directory not provided
03:29:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.831825 4394 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:29:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.832208 4394 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:29:02.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.837944 4394 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/data/instance:
03:29:02.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de8f879ae8454ae79f85f5c3d186b64f"
03:29:02.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:02 on dist-test-slave-qv09"
03:29:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.839053 4394 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/wal/instance:
03:29:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de8f879ae8454ae79f85f5c3d186b64f"
03:29:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:02 on dist-test-slave-qv09"
03:29:02.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.846210 4394 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.001s
03:29:02.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.852049 4410 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:29:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.853200 4394 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
03:29:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.853581 4394 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/data,/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/wal
03:29:02.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de8f879ae8454ae79f85f5c3d186b64f"
03:29:02.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:02 on dist-test-slave-qv09"
03:29:02.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.853995 4394 fs_report.cc:383] FS layout report
03:29:02.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:29:02.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/wal
03:29:02.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/wal
03:29:02.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/data/data
03:29:02.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:29:02.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:29:02.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:29:02.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:29:02.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:29:02.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:29:02.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:29:02.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:29:02.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:29:02.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:29:02.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:29:02.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.899113 4394 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:29:02.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.901073 4394 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:29:02.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.901651 4394 kserver.cc:163] Server-wide thread pool size limit: 3276
03:29:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.979349 4394 rpc_server.cc:307] RPC server started. Bound to: 127.4.71.190:42417
03:29:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.979507 4461 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.71.190:42417 every 8 connection(s)
03:29:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.982184 4394 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/data/info.pb
03:29:02.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.986981 4382 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu as pid 4394
03:29:02.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.987468 4382 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-0/wal/instance
03:29:02.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.988533 4462 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:29:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:02.991622 4382 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/wal
03:29:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/data
03:29:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:29:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:29:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:29:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/logs
03:29:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/data/info.pb
03:29:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:29:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:29:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:29:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:29:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:29:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:29:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.71.189:37207
03:29:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.71.189
03:29:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.71.148:42663
03:29:02.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:02.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:02.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:02.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:02.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.71.190:42417,127.4.71.189:37207,127.4.71.188:40861
03:29:02.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:02.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43105/jwks.json with env {}
03:29:03.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:03.002698 4462 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } has no permanent_uuid. Determining permanent_uuid...
03:29:03.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:03.035144 4462 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } has no permanent_uuid. Determining permanent_uuid...
03:29:03.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.037992 4411 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.71.189:37207: connect: Connection refused (error 111)
03:29:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.042254 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.189:37207: Network error: Client connection negotiation failed: client connection to 127.4.71.189:37207: connect: Connection refused (error 111)
03:29:03.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:03.091768 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } attempt: 1
03:29:03.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.096657 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.189:37207: Network error: Client connection negotiation failed: client connection to 127.4.71.189:37207: connect: Connection refused (error 111)
03:29:03.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:03.165036 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } attempt: 2
03:29:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.168722 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.189:37207: Network error: Client connection negotiation failed: client connection to 127.4.71.189:37207: connect: Connection refused (error 111)
03:29:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:03.260244 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } attempt: 3
03:29:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.273285 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.189:37207: Network error: Client connection negotiation failed: client connection to 127.4.71.189:37207: connect: Connection refused (error 111)
03:29:03.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.333243 4464 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:29:03.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.333806 4464 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:29:03.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.334419 4464 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:29:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.367136 4464 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:29:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.367463 4464 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:29:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.367653 4464 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:29:03.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.367834 4464 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:29:03.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.368009 4464 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:29:03.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.368227 4464 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43105/jwks.json
03:29:03.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:03.403705 4464 master_runner.cc:386] Master server non-default flags:
03:29:03.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:03.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.71.148:42663
03:29:03.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:03.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:03.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/data
03:29:03.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/wal
03:29:03.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:03.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.71.190:42417,127.4.71.189:37207,127.4.71.188:40861
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.71.189:37207
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43105/jwks.json
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/data/info.pb
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.71.189
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:03.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:29:03.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:29:03.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:03.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/logs
03:29:03.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:03.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:29:03.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:29:03.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:29:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:29:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:29:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:29:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:29:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:29:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:29:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:03.404973 4464 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:29:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:03.407143 4464 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:29:03.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:03.416824 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } attempt: 4
03:29:03.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.421898 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.189:37207: Network error: Client connection negotiation failed: client connection to 127.4.71.189:37207: connect: Connection refused (error 111)
03:29:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.427726 4473 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:03.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:03.721347 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } attempt: 5
03:29:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.725916 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.189:37207: Network error: Client connection negotiation failed: client connection to 127.4.71.189:37207: connect: Connection refused (error 111)
03:29:04.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.273401 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } attempt: 6
03:29:04.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:04.282437 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.189:37207: Network error: Client connection negotiation failed: client connection to 127.4.71.189:37207: connect: Connection refused (error 111)
03:29:04.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.427757 4476 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:03.427960 4474 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:04.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:03.430015 4464 server_base.cc:1034] running on GCE node
03:29:04.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.673125 4464 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:29:04.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.675940 4464 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:29:04.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.677350 4464 hybrid_clock.cc:648] HybridClock initialized: now 1790220544677299 us; error 53 us; skew 500 ppm
03:29:04.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.678483 4464 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:29:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.686033 4464 webserver.cc:460] Webserver started at http://127.4.71.189:35843/ using document root <none> and password file <none>
03:29:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.687055 4464 fs_manager.cc:359] Metadata directory not provided
03:29:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.687247 4464 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:29:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.687719 4464 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:29:04.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.693418 4464 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/data/instance:
03:29:04.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "40419f1748144b2dafab2788a9685c14"
03:29:04.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:04 on dist-test-slave-qv09"
03:29:04.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.694641 4464 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/wal/instance:
03:29:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "40419f1748144b2dafab2788a9685c14"
03:29:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:04 on dist-test-slave-qv09"
03:29:04.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.702279 4464 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
03:29:04.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.708393 4485 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:29:04.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.709522 4464 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
03:29:04.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.709990 4464 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/data,/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/wal
03:29:04.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "40419f1748144b2dafab2788a9685c14"
03:29:04.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:04 on dist-test-slave-qv09"
03:29:04.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.710390 4464 fs_report.cc:383] FS layout report
03:29:04.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:29:04.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/wal
03:29:04.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/wal
03:29:04.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/data/data
03:29:04.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:29:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:29:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:29:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:29:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:29:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:29:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:29:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:29:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:29:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:29:04.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:29:04.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.750149 4464 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:29:04.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.751886 4464 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:29:04.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.752434 4464 kserver.cc:163] Server-wide thread pool size limit: 3276
03:29:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.834877 4464 rpc_server.cc:307] RPC server started. Bound to: 127.4.71.189:37207
03:29:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.834957 4536 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.71.189:37207 every 8 connection(s)
03:29:04.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.837908 4464 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/data/info.pb
03:29:04.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.843549 4537 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:29:04.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.847172 4382 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu as pid 4464
03:29:04.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.847681 4382 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-1/wal/instance
03:29:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.851071 4382 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/wal
03:29:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/data
03:29:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:29:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:29:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/logs
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/data/info.pb
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.71.188:40861
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.71.188
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.71.148:42663
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:04.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:04.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.71.190:42417,127.4.71.189:37207,127.4.71.188:40861
03:29:04.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:04.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43105/jwks.json with env {}
03:29:04.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.860526 4537 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } has no permanent_uuid. Determining permanent_uuid...
03:29:04.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.891037 4537 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } has no permanent_uuid. Determining permanent_uuid...
03:29:04.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.908974 4537 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } has no permanent_uuid. Determining permanent_uuid...
03:29:04.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:04.911541 4488 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:04.916613 4537 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:04.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:04.966205 4537 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 1
03:29:04.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:04.970144 4537 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:05.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.038647 4537 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 2
03:29:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.042510 4537 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:05.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.133965 4537 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 3
03:29:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.137719 4537 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:05.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.191629 4539 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:29:05.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.192162 4539 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:29:05.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.192618 4539 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:29:05.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.225991 4539 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:29:05.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.226264 4539 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:29:05.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.226450 4539 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:29:05.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.226641 4539 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:29:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.226895 4539 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:29:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.227057 4539 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43105/jwks.json
03:29:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.261989 4539 master_runner.cc:386] Master server non-default flags:
03:29:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.71.148:42663
03:29:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/data
03:29:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/wal
03:29:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.71.190:42417,127.4.71.189:37207,127.4.71.188:40861
03:29:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.71.188:40861
03:29:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43105/jwks.json
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/data/info.pb
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.71.188
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/logs
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:29:05.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:29:05.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:29:05.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:29:05.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:29:05.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:29:05.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:29:05.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:29:05.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:29:05.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.263716 4539 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:29:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.265933 4539 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:29:05.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.281235 4537 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 4
03:29:05.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.285727 4537 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:05.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.290022 4549 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:05.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.342890 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } attempt: 7
03:29:05.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.359858 4462 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } has no permanent_uuid. Determining permanent_uuid...
03:29:05.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.367048 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.425527 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 1
03:29:05.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.430261 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.511943 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 2
03:29:05.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.516171 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:05.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.585129 4537 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 5
03:29:05.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.590073 4537 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:05.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.601814 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 3
03:29:05.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.607146 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:05.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.747643 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 4
03:29:05.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.752413 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:06.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.035784 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 5
03:29:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:06.040949 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:06.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.137514 4537 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 6
03:29:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:06.145702 4537 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:06.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.290342 4552 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:06.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:05.290531 4550 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:06.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:05.292289 4539 server_base.cc:1034] running on GCE node
03:29:06.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.564990 4539 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:29:06.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.568312 4539 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:29:06.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.569793 4539 hybrid_clock.cc:648] HybridClock initialized: now 1790220546569717 us; error 86 us; skew 500 ppm
03:29:06.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.570681 4539 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:29:06.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.586886 4539 webserver.cc:460] Webserver started at http://127.4.71.188:34863/ using document root <none> and password file <none>
03:29:06.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.587924 4539 fs_manager.cc:359] Metadata directory not provided
03:29:06.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.588125 4539 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:29:06.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.588531 4539 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:29:06.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.593365 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 6
03:29:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.594367 4539 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/data/instance:
03:29:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8f30822399d54dfea0dfee13a5f06248"
03:29:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:06 on dist-test-slave-qv09"
03:29:06.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.596020 4539 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/wal/instance:
03:29:06.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8f30822399d54dfea0dfee13a5f06248"
03:29:06.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:06 on dist-test-slave-qv09"
03:29:06.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:06.600258 4462 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.71.188:40861: Network error: Client connection negotiation failed: client connection to 127.4.71.188:40861: connect: Connection refused (error 111)
03:29:06.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.606542 4539 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.008s sys 0.000s
03:29:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.614691 4565 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:29:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.616187 4539 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.004s
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.616619 4539 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/data,/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/wal
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8f30822399d54dfea0dfee13a5f06248"
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:06 on dist-test-slave-qv09"
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.617108 4539 fs_report.cc:383] FS layout report
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/wal
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/wal
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/data/data
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:29:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:29:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:29:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:29:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:29:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:29:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.657902 4539 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:29:06.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.659583 4539 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:29:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.660019 4539 kserver.cc:163] Server-wide thread pool size limit: 3276
03:29:06.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.741674 4539 rpc_server.cc:307] RPC server started. Bound to: 127.4.71.188:40861
03:29:06.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.741679 4616 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.71.188:40861 every 8 connection(s)
03:29:06.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.744604 4539 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/data/info.pb
03:29:06.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.751191 4617 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:29:06.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.752270 4382 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu as pid 4539
03:29:06.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.752727 4382 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/master-2/wal/instance
03:29:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.770361 4617 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } has no permanent_uuid. Determining permanent_uuid...
03:29:06.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.805465 4617 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } has no permanent_uuid. Determining permanent_uuid...
03:29:06.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.819587 4617 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } has no permanent_uuid. Determining permanent_uuid...
03:29:06.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.841331 4617 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248: Bootstrap starting.
03:29:06.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.850087 4617 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248: Neither blocks nor log segments found. Creating new log.
03:29:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.852905 4617 log.cc:824] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248: Log is configured to *not* fsync() on all Append() calls
03:29:06.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.861150 4617 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248: No bootstrap required, opened a new log
03:29:06.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.887439 4617 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } }
03:29:06.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.888065 4617 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:29:06.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.888262 4617 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f30822399d54dfea0dfee13a5f06248, State: Initialized, Role: FOLLOWER
03:29:06.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.888935 4617 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } }
03:29:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.891722 4626 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } } }
03:29:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.892474 4626 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [sys.catalog]: This master's current role is: FOLLOWER
03:29:06.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.893483 4617 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [sys.catalog]: configured and running, proceeding with master startup.
03:29:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:06.916180 4571 tablet.cc:2369] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:29:06.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:06.923745 4637 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:06.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:06.924088 4637 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:06.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.919076 4539 master_runner.cc:186] Error getting master registration for 127.4.71.190:42417: OK, instance_id { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" instance_seqno: 1790220542958291 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:29:06.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:06.948707 4539 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:29:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.033809 4626 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:29:07.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.034330 4626 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } }
03:29:07.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.036758 4626 leader_election.cc:288] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers de8f879ae8454ae79f85f5c3d186b64f (127.4.71.190:42417), 40419f1748144b2dafab2788a9685c14 (127.4.71.189:37207)
03:29:07.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.206297 4537 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 7
03:29:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.230633 4537 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14: Bootstrap starting.
03:29:07.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.239284 4537 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14: Neither blocks nor log segments found. Creating new log.
03:29:07.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.241986 4537 log.cc:824] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14: Log is configured to *not* fsync() on all Append() calls
03:29:07.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.246994 4537 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14: No bootstrap required, opened a new log
03:29:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.278919 4537 raft_consensus.cc:348] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } }
03:29:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.279850 4537 raft_consensus.cc:374] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:29:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.280162 4537 raft_consensus.cc:729] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 40419f1748144b2dafab2788a9685c14, State: Initialized, Role: FOLLOWER
03:29:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.281090 4537 consensus_queue.cc:260] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } }
03:29:07.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.284114 4641 sys_catalog.cc:455] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } } }
03:29:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.284957 4641 sys_catalog.cc:458] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [sys.catalog]: This master's current role is: FOLLOWER
03:29:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.285924 4537 sys_catalog.cc:564] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [sys.catalog]: configured and running, proceeding with master startup.
03:29:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.286108 4512 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8f30822399d54dfea0dfee13a5f06248" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "40419f1748144b2dafab2788a9685c14" is_pre_election: true
03:29:07.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.287402 4512 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8f30822399d54dfea0dfee13a5f06248 in term 0.
03:29:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.288803 4566 leader_election.cc:302] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 40419f1748144b2dafab2788a9685c14, 8f30822399d54dfea0dfee13a5f06248; no voters:
03:29:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.289803 4626 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:29:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.290154 4626 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:29:07.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.290902 4626 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [term 0 FOLLOWER]: Advancing to term 1
03:29:07.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:07.325359 4652 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:07.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:07.325688 4652 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.321736 4464 master_runner.cc:186] Error getting master registration for 127.4.71.190:42417: OK, instance_id { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" instance_seqno: 1790220542958291 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:29:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.355505 4464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:29:07.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.360949 4626 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } }
03:29:07.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.362946 4626 leader_election.cc:288] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [CANDIDATE]: Term 1 election: Requested vote from peers de8f879ae8454ae79f85f5c3d186b64f (127.4.71.190:42417), 40419f1748144b2dafab2788a9685c14 (127.4.71.189:37207)
03:29:07.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.366572 4512 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8f30822399d54dfea0dfee13a5f06248" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "40419f1748144b2dafab2788a9685c14"
03:29:07.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.367178 4512 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [term 0 FOLLOWER]: Advancing to term 1
03:29:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.465251 4512 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f30822399d54dfea0dfee13a5f06248 in term 1.
03:29:07.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.472091 4566 leader_election.cc:302] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 40419f1748144b2dafab2788a9685c14, 8f30822399d54dfea0dfee13a5f06248; no voters:
03:29:07.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.472944 4626 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [term 1 FOLLOWER]: Leader election won for term 1
03:29:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.489214 4626 raft_consensus.cc:686] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [term 1 LEADER]: Becoming Leader. State: Replica: 8f30822399d54dfea0dfee13a5f06248, State: Running, Role: LEADER
03:29:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.490720 4626 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } }
03:29:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.498922 4654 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8f30822399d54dfea0dfee13a5f06248. Latest consensus state: current_term: 1 leader_uuid: "8f30822399d54dfea0dfee13a5f06248" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } } }
03:29:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.500074 4654 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [sys.catalog]: This master's current role is: LEADER
03:29:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.507535 4656 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
03:29:07.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:07.512050 4491 tablet.cc:2369] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:29:07.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.520761 4656 catalog_manager.cc:1484] Initializing Kudu cluster ID...
03:29:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.542150 4512 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [term 1 FOLLOWER]: Refusing update from remote peer 8f30822399d54dfea0dfee13a5f06248: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:29:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.545288 4626 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [LEADER]: Connected to new peer: Peer: permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:29:07.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.580125 4641 sys_catalog.cc:455] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8f30822399d54dfea0dfee13a5f06248. Latest consensus state: current_term: 1 leader_uuid: "8f30822399d54dfea0dfee13a5f06248" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } } }
03:29:07.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.580976 4641 sys_catalog.cc:458] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [sys.catalog]: This master's current role is: FOLLOWER
03:29:07.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.588779 4654 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8f30822399d54dfea0dfee13a5f06248" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } } }
03:29:07.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.589588 4654 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [sys.catalog]: This master's current role is: LEADER
03:29:07.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.592949 4641 sys_catalog.cc:455] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8f30822399d54dfea0dfee13a5f06248" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } } }
03:29:07.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.594818 4641 sys_catalog.cc:458] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14 [sys.catalog]: This master's current role is: FOLLOWER
03:29:07.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.595412 4656 catalog_manager.cc:1347] Generated new cluster ID: 8ca8724feb6b40e99625c60d95bb0415
03:29:07.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.595690 4656 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
03:29:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.633678 4462 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } attempt: 7
03:29:07.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.647404 4656 catalog_manager.cc:1370] Generated new certificate authority record
03:29:07.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.650415 4656 catalog_manager.cc:1504] Loading token signing keys...
03:29:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.682531 4462 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f: Bootstrap starting.
03:29:07.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.698210 4462 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f: Neither blocks nor log segments found. Creating new log.
03:29:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.701331 4462 log.cc:824] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f: Log is configured to *not* fsync() on all Append() calls
03:29:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.714743 4462 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f: No bootstrap required, opened a new log
03:29:07.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.722168 4656 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248: Generated new TSK 0
03:29:07.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.726385 4656 catalog_manager.cc:1514] Initializing in-progress tserver states...
03:29:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.757169 4462 raft_consensus.cc:348] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } }
03:29:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.758095 4462 raft_consensus.cc:374] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:29:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.758595 4462 raft_consensus.cc:729] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: de8f879ae8454ae79f85f5c3d186b64f, State: Initialized, Role: FOLLOWER
03:29:07.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.759542 4462 consensus_queue.cc:260] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } }
03:29:07.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.764248 4462 sys_catalog.cc:564] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f [sys.catalog]: configured and running, proceeding with master startup.
03:29:07.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.764866 4437 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8f30822399d54dfea0dfee13a5f06248" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "de8f879ae8454ae79f85f5c3d186b64f" is_pre_election: true
03:29:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.765705 4435 raft_consensus.cc:3037] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f [term 0 FOLLOWER]: Advancing to term 1
03:29:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.765826 4436 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8f30822399d54dfea0dfee13a5f06248" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "de8f879ae8454ae79f85f5c3d186b64f"
03:29:07.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.780722 4435 raft_consensus.cc:1261] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f [term 1 FOLLOWER]: Refusing update from remote peer 8f30822399d54dfea0dfee13a5f06248: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:29:07.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.763545 4664 sys_catalog.cc:455] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } } }
03:29:07.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.783386 4626 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [LEADER]: Connected to new peer: Peer: permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:29:07.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.782924 4664 sys_catalog.cc:458] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f [sys.catalog]: This master's current role is: FOLLOWER
03:29:07.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.787946 4382 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:07.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:07.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/wal
03:29:07.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/data
03:29:07.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:29:07.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:29:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:29:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/logs
03:29:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/data/info.pb
03:29:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:29:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.71.129:0
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.71.129
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.71.129
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.71.190:42417,127.4.71.189:37207,127.4.71.188:40861
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.71.148:42663
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:29:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.872759 4672 sys_catalog.cc:455] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8f30822399d54dfea0dfee13a5f06248" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } } }
03:29:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.873749 4672 sys_catalog.cc:458] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f [sys.catalog]: This master's current role is: FOLLOWER
03:29:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.874960 4664 sys_catalog.cc:455] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8f30822399d54dfea0dfee13a5f06248. Latest consensus state: current_term: 1 leader_uuid: "8f30822399d54dfea0dfee13a5f06248" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } } }
03:29:07.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.875936 4664 sys_catalog.cc:458] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f [sys.catalog]: This master's current role is: FOLLOWER
03:29:07.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:07.877588 4681 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:29:08.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:08.331820 4652 catalog_manager.cc:1259] Loaded cluster ID: 8ca8724feb6b40e99625c60d95bb0415
03:29:08.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:08.332470 4652 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14: loading cluster ID for follower catalog manager: success
03:29:08.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:08.340642 4667 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:29:08.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:08.341445 4652 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14: acquiring CA information for follower catalog manager: success
03:29:08.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:08.341555 4667 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:29:08.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:08.342296 4667 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:29:08.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:08.347339 4652 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 40419f1748144b2dafab2788a9685c14: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:29:08.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:08.396853 4667 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:29:08.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:08.398255 4667 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.71.129
03:29:08.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:08.400676 4686 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8f30822399d54dfea0dfee13a5f06248" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "de8f879ae8454ae79f85f5c3d186b64f" member_type: VOTER last_known_addr { host: "127.4.71.190" port: 42417 } } peers { permanent_uuid: "40419f1748144b2dafab2788a9685c14" member_type: VOTER last_known_addr { host: "127.4.71.189" port: 37207 } } peers { permanent_uuid: "8f30822399d54dfea0dfee13a5f06248" member_type: VOTER last_known_addr { host: "127.4.71.188" port: 40861 } } }
03:29:08.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:08.401705 4686 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8f30822399d54dfea0dfee13a5f06248 [sys.catalog]: This master's current role is: LEADER
03:29:08.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:08.441431 4667 tablet_server_runner.cc:78] Tablet server non-default flags:
03:29:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.71.148:42663
03:29:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/data
03:29:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/wal
03:29:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.71.129:0
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/data/info.pb
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.71.129
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.71.190:42417,127.4.71.189:37207,127.4.71.188:40861
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.71.129
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/logs
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:29:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:29:08.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:29:08.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:08.442989 4667 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:29:08.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:08.444996 4667 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:29:08.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:08.472783 4691 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:08.899734 4681 catalog_manager.cc:1259] Loaded cluster ID: 8ca8724feb6b40e99625c60d95bb0415
03:29:08.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:08.903138 4681 catalog_manager.cc:1552] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f: loading cluster ID for follower catalog manager: success
03:29:08.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:08.924795 4681 catalog_manager.cc:1574] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f: acquiring CA information for follower catalog manager: success
03:29:08.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:08.947800 4681 catalog_manager.cc:1602] T 00000000000000000000000000000000 P de8f879ae8454ae79f85f5c3d186b64f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:29:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:09.862659 4689 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 4667
03:29:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:10.387714 4689 kernel_stack_watchdog.cc:198] Thread 4667 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 390ms:
03:29:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:29:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:29:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:29:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:29:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:08.474188 4690 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:10.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:10.393383 4667 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.924s user 0.629s sys 1.134s
03:29:10.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:10.393733 4667 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.924s user 0.629s sys 1.134s
03:29:10.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:10.399906 4696 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:10.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.401480 4667 server_base.cc:1034] running on GCE node
03:29:10.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.402810 4667 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:29:10.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.405575 4667 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:29:10.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.407025 4667 hybrid_clock.cc:648] HybridClock initialized: now 1790220550406819 us; error 215 us; skew 500 ppm
03:29:10.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.407917 4667 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:29:10.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.415767 4667 webserver.cc:460] Webserver started at http://127.4.71.129:41637/ using document root <none> and password file <none>
03:29:10.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.416889 4667 fs_manager.cc:359] Metadata directory not provided
03:29:10.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.417138 4667 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:29:10.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.417695 4667 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:29:10.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.424144 4667 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/data/instance:
03:29:10.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c76bbd210abe4cd9b96af31ada2acf46"
03:29:10.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:10 on dist-test-slave-qv09"
03:29:10.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.425361 4667 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/wal/instance:
03:29:10.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c76bbd210abe4cd9b96af31ada2acf46"
03:29:10.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:10 on dist-test-slave-qv09"
03:29:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.434511 4667 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.003s
03:29:10.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.442687 4701 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:29:10.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.444005 4667 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.004s
03:29:10.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.444401 4667 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/data,/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/wal
03:29:10.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c76bbd210abe4cd9b96af31ada2acf46"
03:29:10.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:10 on dist-test-slave-qv09"
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.444780 4667 fs_report.cc:383] FS layout report
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/wal
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/wal
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/data/data
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:29:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:29:10.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.492486 4667 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:29:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.494428 4667 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:29:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.495007 4667 kserver.cc:163] Server-wide thread pool size limit: 3276
03:29:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.498430 4667 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:29:10.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.505084 4667 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:29:10.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.505384 4667 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:29:10.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.505682 4667 ts_tablet_manager.cc:610] Registered 0 tablets
03:29:10.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.505895 4667 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:29:10.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.706923 4667 rpc_server.cc:307] RPC server started. Bound to: 127.4.71.129:44557
03:29:10.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.707021 4813 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.71.129:44557 every 8 connection(s)
03:29:10.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.709776 4667 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/data/info.pb
03:29:10.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.718964 4382 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu as pid 4667
03:29:10.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.719573 4382 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-0/wal/instance
03:29:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.762076 4382 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/wal
03:29:10.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/data
03:29:10.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:29:10.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/logs
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/data/info.pb
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:10.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:29:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:29:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.71.130:0
03:29:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.71.130
03:29:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.71.130
03:29:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.71.190:42417,127.4.71.189:37207,127.4.71.188:40861
03:29:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.71.148:42663
03:29:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:29:10.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.774389 4814 heartbeater.cc:347] Connected to a master server at 127.4.71.189:37207
03:29:10.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.775403 4814 heartbeater.cc:464] Registering TS with master...
03:29:10.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.776775 4814 heartbeater.cc:511] Master 127.4.71.189:37207 requested a full tablet report, sending...
03:29:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.781431 4815 heartbeater.cc:347] Connected to a master server at 127.4.71.188:40861
03:29:10.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.781791 4815 heartbeater.cc:464] Registering TS with master...
03:29:10.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.783169 4815 heartbeater.cc:511] Master 127.4.71.188:40861 requested a full tablet report, sending...
03:29:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.784672 4502 ts_manager.cc:194] Registered new tserver with Master: c76bbd210abe4cd9b96af31ada2acf46 (127.4.71.129:44557)
03:29:10.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.789093 4582 ts_manager.cc:194] Registered new tserver with Master: c76bbd210abe4cd9b96af31ada2acf46 (127.4.71.129:44557)
03:29:10.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.797341 4582 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.71.129:38123
03:29:10.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.825950 4816 heartbeater.cc:347] Connected to a master server at 127.4.71.190:42417
03:29:10.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.826344 4816 heartbeater.cc:464] Registering TS with master...
03:29:10.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.827028 4816 heartbeater.cc:511] Master 127.4.71.190:42417 requested a full tablet report, sending...
03:29:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:10.830026 4427 ts_manager.cc:194] Registered new tserver with Master: c76bbd210abe4cd9b96af31ada2acf46 (127.4.71.129:44557)
03:29:11.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:11.168229 4825 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:29:11.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:11.168805 4825 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:29:11.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:11.169431 4825 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:29:11.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:11.202739 4825 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:29:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:11.203896 4825 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.71.130
03:29:11.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:11.241461 4825 tablet_server_runner.cc:78] Tablet server non-default flags:
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.71.148:42663
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/data
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/wal
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.71.130:0
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/data/info.pb
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.71.130
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.71.190:42417,127.4.71.189:37207,127.4.71.188:40861
03:29:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:29:11.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.71.130
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/logs
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:29:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:11.242982 4825 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:29:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:11.245066 4825 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:29:11.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:11.271922 4835 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:11.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:11.804384 4815 heartbeater.cc:503] Master 127.4.71.188:40861 was elected leader, sending a full tablet report...
03:29:12.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:11.272511 4832 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:12.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:11.274425 4833 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:12.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:12.525738 4834 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:29:12.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.525892 4825 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:29:12.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.530217 4825 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:29:12.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.532687 4825 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:29:12.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.534054 4825 hybrid_clock.cc:648] HybridClock initialized: now 1790220552534020 us; error 76 us; skew 500 ppm
03:29:12.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.534981 4825 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:29:12.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.542390 4825 webserver.cc:460] Webserver started at http://127.4.71.130:40103/ using document root <none> and password file <none>
03:29:12.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.543407 4825 fs_manager.cc:359] Metadata directory not provided
03:29:12.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.543622 4825 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:29:12.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.544031 4825 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:29:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.549183 4825 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/data/instance:
03:29:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e89bf6997dd44e4a21ab75bf3981cec"
03:29:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:12 on dist-test-slave-qv09"
03:29:12.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.550324 4825 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/wal/instance:
03:29:12.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e89bf6997dd44e4a21ab75bf3981cec"
03:29:12.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:12 on dist-test-slave-qv09"
03:29:12.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.558228 4825 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
03:29:12.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.564394 4843 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:29:12.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.565689 4825 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
03:29:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.565979 4825 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/data,/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/wal
03:29:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e89bf6997dd44e4a21ab75bf3981cec"
03:29:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:12 on dist-test-slave-qv09"
03:29:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.566294 4825 fs_report.cc:383] FS layout report
03:29:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:29:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/wal
03:29:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/wal
03:29:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/data/data
03:29:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:29:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:29:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:29:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:29:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:29:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:29:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:29:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:29:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:29:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:29:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:29:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.600253 4825 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:29:12.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.601662 4825 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:29:12.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.602082 4825 kserver.cc:163] Server-wide thread pool size limit: 3276
03:29:12.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.605170 4825 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:29:12.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.609336 4825 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:29:12.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.609525 4825 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:29:12.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.609726 4825 ts_tablet_manager.cc:610] Registered 0 tablets
03:29:12.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.609861 4825 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:29:12.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.773993 4825 rpc_server.cc:307] RPC server started. Bound to: 127.4.71.130:41417
03:29:12.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.774067 4955 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.71.130:41417 every 8 connection(s)
03:29:12.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.776518 4825 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/data/info.pb
03:29:12.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.779381 4382 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu as pid 4825
03:29:12.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.780509 4382 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-1/wal/instance
03:29:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.803721 4382 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/wal
03:29:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/data
03:29:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:29:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:29:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:29:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/logs
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/data/info.pb
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.71.131:0
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.71.131
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.71.131
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.71.190:42417,127.4.71.189:37207,127.4.71.188:40861
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.71.148:42663
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:12.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:29:12.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.848219 4956 heartbeater.cc:347] Connected to a master server at 127.4.71.189:37207
03:29:12.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.848697 4956 heartbeater.cc:464] Registering TS with master...
03:29:12.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.850288 4956 heartbeater.cc:511] Master 127.4.71.189:37207 requested a full tablet report, sending...
03:29:12.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.853372 4502 ts_manager.cc:194] Registered new tserver with Master: 2e89bf6997dd44e4a21ab75bf3981cec (127.4.71.130:41417)
03:29:12.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.857656 4957 heartbeater.cc:347] Connected to a master server at 127.4.71.188:40861
03:29:12.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.858014 4957 heartbeater.cc:464] Registering TS with master...
03:29:12.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.858868 4957 heartbeater.cc:511] Master 127.4.71.188:40861 requested a full tablet report, sending...
03:29:12.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.861505 4582 ts_manager.cc:194] Registered new tserver with Master: 2e89bf6997dd44e4a21ab75bf3981cec (127.4.71.130:41417)
03:29:12.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.863466 4582 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.71.130:46609
03:29:12.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.872305 4958 heartbeater.cc:347] Connected to a master server at 127.4.71.190:42417
03:29:12.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.872671 4958 heartbeater.cc:464] Registering TS with master...
03:29:12.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.873349 4958 heartbeater.cc:511] Master 127.4.71.190:42417 requested a full tablet report, sending...
03:29:12.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:12.876698 4427 ts_manager.cc:194] Registered new tserver with Master: 2e89bf6997dd44e4a21ab75bf3981cec (127.4.71.130:41417)
03:29:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:13.175084 4966 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:29:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:13.175576 4966 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:29:13.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:13.176144 4966 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:29:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:13.208062 4966 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:29:13.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:13.208875 4966 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.71.131
03:29:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:13.244517 4966 tablet_server_runner.cc:78] Tablet server non-default flags:
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.71.148:42663
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/data
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/wal
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.71.131:0
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/data/info.pb
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.71.131
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.71.190:42417,127.4.71.189:37207,127.4.71.188:40861
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.71.131
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/logs
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:29:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:13.245678 4966 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:29:13.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:13.247388 4966 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:29:13.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:13.271142 4972 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:13.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:13.869230 4957 heartbeater.cc:503] Master 127.4.71.188:40861 was elected leader, sending a full tablet report...
03:29:14.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:13.272250 4973 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:14.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:13.273978 4966 server_base.cc:1034] running on GCE node
03:29:14.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:13.273377 4975 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:14.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.484081 4966 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:29:14.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.486548 4966 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:29:14.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.487967 4966 hybrid_clock.cc:648] HybridClock initialized: now 1790220554487938 us; error 43 us; skew 500 ppm
03:29:14.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.488976 4966 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:29:14.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.495405 4966 webserver.cc:460] Webserver started at http://127.4.71.131:41443/ using document root <none> and password file <none>
03:29:14.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.496394 4966 fs_manager.cc:359] Metadata directory not provided
03:29:14.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.496646 4966 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:29:14.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.497171 4966 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:29:14.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.502259 4966 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/data/instance:
03:29:14.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df49059e28354969bf7ee46a0b88dfc7"
03:29:14.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:14 on dist-test-slave-qv09"
03:29:14.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.503595 4966 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/wal/instance:
03:29:14.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df49059e28354969bf7ee46a0b88dfc7"
03:29:14.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:14 on dist-test-slave-qv09"
03:29:14.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.510914 4966 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
03:29:14.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.516429 4983 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:29:14.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.517652 4966 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
03:29:14.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.518007 4966 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/data,/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/wal
03:29:14.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df49059e28354969bf7ee46a0b88dfc7"
03:29:14.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:14 on dist-test-slave-qv09"
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.518488 4966 fs_report.cc:383] FS layout report
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/wal
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/wal
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/data/data
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:29:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:29:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.558686 4966 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:29:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.560163 4966 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:29:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.560583 4966 kserver.cc:163] Server-wide thread pool size limit: 3276
03:29:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.562985 4966 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:29:14.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.567672 4966 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:29:14.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.567934 4966 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:29:14.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.568140 4966 ts_tablet_manager.cc:610] Registered 0 tablets
03:29:14.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.568279 4966 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:29:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.726167 4966 rpc_server.cc:307] RPC server started. Bound to: 127.4.71.131:36931
03:29:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.726754 5095 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.71.131:36931 every 8 connection(s)
03:29:14.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.728725 4966 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/data/info.pb
03:29:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.729858 4382 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu as pid 4966
03:29:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.730285 4382 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster3223857469430862563/ts-2/wal/instance
03:29:14.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.776521 5096 heartbeater.cc:347] Connected to a master server at 127.4.71.189:37207
03:29:14.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.777057 5096 heartbeater.cc:464] Registering TS with master...
03:29:14.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.778386 5096 heartbeater.cc:511] Master 127.4.71.189:37207 requested a full tablet report, sending...
03:29:14.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.781502 5097 heartbeater.cc:347] Connected to a master server at 127.4.71.188:40861
03:29:14.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.781826 5097 heartbeater.cc:464] Registering TS with master...
03:29:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.783064 5097 heartbeater.cc:511] Master 127.4.71.188:40861 requested a full tablet report, sending...
03:29:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.783450 5098 heartbeater.cc:347] Connected to a master server at 127.4.71.190:42417
03:29:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.783769 5098 heartbeater.cc:464] Registering TS with master...
03:29:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.782965 4502 ts_manager.cc:194] Registered new tserver with Master: df49059e28354969bf7ee46a0b88dfc7 (127.4.71.131:36931)
03:29:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.784586 5098 heartbeater.cc:511] Master 127.4.71.190:42417 requested a full tablet report, sending...
03:29:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.786473 4582 ts_manager.cc:194] Registered new tserver with Master: df49059e28354969bf7ee46a0b88dfc7 (127.4.71.131:36931)
03:29:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.788407 4582 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.71.131:46313
03:29:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.789947 4427 ts_manager.cc:194] Registered new tserver with Master: df49059e28354969bf7ee46a0b88dfc7 (127.4.71.131:36931)
03:29:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:14.795408 4382 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
03:29:14.807 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:29:14.808 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:29:14.820 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.71.190"
port: 42417
}
bound_http_address {
host: "127.4.71.190"
port: 36265
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.71.189"
port: 37207
}
bound_http_address {
host: "127.4.71.189"
port: 35843
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.71.188"
port: 40861
}
bound_http_address {
host: "127.4.71.188"
port: 34863
}
}
}
03:29:14.825 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:29:14.827 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.4.71.129"
port: 44557
}
bound_http_address {
host: "127.4.71.129"
port: 41637
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.4.71.130"
port: 41417
}
bound_http_address {
host: "127.4.71.130"
port: 40103
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.4.71.131"
port: 36931
}
bound_http_address {
host: "127.4.71.131"
port: 41443
}
}
}
03:29:14.828 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:29:15.030 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:29:15.153 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskK6vj3Y/test-tmp
03:29:15.154 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:29:15.154 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskK6vj3Y/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:29:15.159 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
03:29:15.500 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:29:15.500 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:29:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:15.503661 5107 mini_oidc.cc:152] Starting JWKS server
03:29:15.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:15.508168 5107 webserver.cc:460] Webserver started at http://127.0.0.1:42037/ using document root <none> and password file <none>
03:29:15.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:15.508737 5107 mini_oidc.cc:189] Starting OIDC Discovery server
03:29:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:15.510456 5107 webserver.cc:460] Webserver started at http://127.0.0.1:43533/ using document root <none> and password file <none>
03:29:15.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
03:29:15.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/principal' for realm 'KRBTEST.COM',
03:29:15.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
03:29:15.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:15 dist-test-slave-qv09 krb5kdc[5120](info): setting up network...
03:29:15.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
03:29:15.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:15 dist-test-slave-qv09 krb5kdc[5120](info): set up 2 sockets
03:29:15.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:15 dist-test-slave-qv09 krb5kdc[5120](info): commencing operation
03:29:15.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
03:29:15.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:15.792537 5097 heartbeater.cc:503] Master 127.4.71.188:40861 was elected leader, sending a full tablet report...
03:29:16.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:16.629280 5107 mini_kdc.cc:121] Time spent starting KDC: real 1.093s user 0.001s sys 0.019s
03:29:16.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
03:29:16.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:29:16.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
03:29:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
03:29:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:29:16.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
03:29:16.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
03:29:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:29:16.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
03:29:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:29:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/test-user.keytab.
03:29:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/test-user.keytab.
03:29:16.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:16 dist-test-slave-qv09 krb5kdc[5120](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220556, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:29:16.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:29:16.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T03:29:16Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:29:16.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T03:29:16Z Disabled control of system clock
03:29:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.252.254@KRBTEST.COM; defaulting to no policy
03:29:16.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:29:16.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.252.254@KRBTEST.COM" created.
03:29:16.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:29:16.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.252.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.254.keytab.
03:29:16.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.252.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.254.keytab.
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:16.851799 5107 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/logs
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data/info.pb
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:16.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.252.254:34807
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.252.254
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.254.keytab
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.252.254
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.252.212:33377
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773
03:29:16.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:16.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42037/jwks.json
03:29:16.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:29:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:17.170334 5133 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:29:17.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:17.170879 5133 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:29:17.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:17.171554 5133 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:29:17.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:17.218261 5133 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:29:17.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:17.218580 5133 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:29:17.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:17.219039 5133 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:29:17.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:17.219447 5133 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:29:17.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:17.219967 5133 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:29:17.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:17.220439 5133 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:29:17.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:17.220813 5133 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42037/jwks.json
03:29:17.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:17.269259 5133 master_runner.cc:386] Master server non-default flags:
03:29:17.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:17.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.252.212:33377
03:29:17.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:17.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:17.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.254.keytab
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.252.254
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.252.254:34807
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42037/jwks.json
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data/info.pb
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.252.254
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/logs
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:29:17.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:17.270592 5133 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:29:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:17.271649 5133 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:29:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:17.289848 5140 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:18.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:18.231519 4533 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0be0 after lost signal to thread 4468
03:29:18.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:18.232206 4533 debug-util.cc:398] Leaking SignalData structure 0x7b08000ade20 after lost signal to thread 4536
03:29:18.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:17.289909 5139 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:18.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:17.290148 5142 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:18.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:18.552825 5141 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:29:18.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.553488 5133 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:29:18.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.555091 5133 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:29:18.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.558789 5133 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:29:18.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.560487 5133 hybrid_clock.cc:648] HybridClock initialized: now 1790220558560422 us; error 51 us; skew 500 ppm
03:29:18.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.561642 5133 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:29:18.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:18 dist-test-slave-qv09 krb5kdc[5120](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220558, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:29:18.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.575429 5133 init.cc:377] Logged in from keytab as kudu/127.4.252.254@KRBTEST.COM (short username kudu)
03:29:18.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.582332 5133 webserver.cc:460] Webserver started at http://127.4.252.254:32999/ using document root <none> and password file <none>
03:29:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.583374 5133 fs_manager.cc:359] Metadata directory not provided
03:29:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.583591 5133 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:29:18.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.584015 5133 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:29:18.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.589735 5133 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data/instance:
03:29:18.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f755225b6a754d71b231fcbebbe70e37"
03:29:18.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:18 on dist-test-slave-qv09"
03:29:18.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.591028 5133 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal/instance:
03:29:18.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f755225b6a754d71b231fcbebbe70e37"
03:29:18.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:18 on dist-test-slave-qv09"
03:29:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.600208 5133 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
03:29:18.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.607465 5151 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:29:18.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.608767 5133 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
03:29:18.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.609099 5133 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data,/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal
03:29:18.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f755225b6a754d71b231fcbebbe70e37"
03:29:18.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:18 on dist-test-slave-qv09"
03:29:18.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.609436 5133 fs_report.cc:383] FS layout report
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data/data
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:29:18.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:29:18.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.665261 5133 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:29:18.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.671763 5133 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:29:18.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.672179 5133 kserver.cc:163] Server-wide thread pool size limit: 3276
03:29:18.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.796252 5133 rpc_server.cc:307] RPC server started. Bound to: 127.4.252.254:34807
03:29:18.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.796303 5202 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.252.254:34807 every 8 connection(s)
03:29:18.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.800746 5133 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data/info.pb
03:29:18.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.811518 5107 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu as pid 5133
03:29:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.811869 5203 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:29:18.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.812098 5107 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal/instance
03:29:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.827668 5203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } has no permanent_uuid. Determining permanent_uuid...
03:29:18.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.252.253@KRBTEST.COM; defaulting to no policy
03:29:18.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:29:18.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.252.253@KRBTEST.COM" created.
03:29:18.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:18 dist-test-slave-qv09 krb5kdc[5120](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220558, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.254@KRBTEST.COM for kudu/127.4.252.254@KRBTEST.COM
03:29:18.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:29:18.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.252.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.253.keytab.
03:29:18.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.252.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.253.keytab.
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.879980 5107 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/logs
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data/info.pb
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.252.253:42165
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.252.253
03:29:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.253.keytab
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.252.253
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.252.212:33377
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42037/jwks.json
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.880584 5203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } has no permanent_uuid. Determining permanent_uuid...
03:29:18.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:18.883790 5153 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:18.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:18.894480 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.253:42165: Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:18.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:18.944135 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } attempt: 1
03:29:18.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:18.948588 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.253:42165: Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:19.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:19.017158 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } attempt: 2
03:29:19.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.022122 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.253:42165: Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:19.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:19.113595 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } attempt: 3
03:29:19.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.118029 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.253:42165: Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:19.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.240394 5210 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:29:19.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.241108 5210 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:29:19.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.242197 5210 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:29:19.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:19.261488 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } attempt: 4
03:29:19.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.269850 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.253:42165: Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:19.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.280881 5210 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:29:19.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.281124 5210 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:29:19.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.281381 5210 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:29:19.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.281579 5210 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:29:19.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.281765 5210 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:29:19.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.281949 5210 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:29:19.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.282094 5210 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42037/jwks.json
03:29:19.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:19.320430 5210 master_runner.cc:386] Master server non-default flags:
03:29:19.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.252.212:33377
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.253.keytab
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.252.253
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.252.253:42165
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42037/jwks.json
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data/info.pb
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.252.253
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:29:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:29:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:29:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/logs
03:29:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:29:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:29:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:29:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:29:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:29:19.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:29:19.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:29:19.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:29:19.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:29:19.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:19.321655 5210 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:29:19.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:19.322584 5210 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:29:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.346453 5217 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:19.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:19.570485 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } attempt: 5
03:29:19.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.576292 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.253:42165: Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:20.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.123750 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } attempt: 6
03:29:20.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:20.159569 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.253:42165: Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.346547 5219 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:19.347905 5216 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:20.838927 5218 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1491 milliseconds
03:29:20.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.839046 5210 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:29:20.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.840482 5210 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:29:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.847705 5210 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:29:20.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.849164 5210 hybrid_clock.cc:648] HybridClock initialized: now 1790220560849118 us; error 44 us; skew 500 ppm
03:29:20.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.850140 5210 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:29:20.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:20 dist-test-slave-qv09 krb5kdc[5120](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220560, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:29:20.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.857105 5210 init.cc:377] Logged in from keytab as kudu/127.4.252.253@KRBTEST.COM (short username kudu)
03:29:20.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.862931 5210 webserver.cc:460] Webserver started at http://127.4.252.253:43345/ using document root <none> and password file <none>
03:29:20.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.864101 5210 fs_manager.cc:359] Metadata directory not provided
03:29:20.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.864351 5210 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:29:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.864881 5210 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:29:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.871016 5210 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data/instance:
03:29:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2f82d9c20a94427b4f40a49cc87a0c0"
03:29:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:20 on dist-test-slave-qv09"
03:29:20.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.872172 5210 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal/instance:
03:29:20.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2f82d9c20a94427b4f40a49cc87a0c0"
03:29:20.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:20 on dist-test-slave-qv09"
03:29:20.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.880831 5210 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
03:29:20.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.887120 5232 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:29:20.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.888832 5210 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
03:29:20.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.889230 5210 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data,/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal
03:29:20.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2f82d9c20a94427b4f40a49cc87a0c0"
03:29:20.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:20 on dist-test-slave-qv09"
03:29:20.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.889599 5210 fs_report.cc:383] FS layout report
03:29:20.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:29:20.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal
03:29:20.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal
03:29:20.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data/data
03:29:20.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:29:20.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:29:20.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:29:20.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:29:20.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:29:20.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:29:20.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:29:20.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:29:20.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:29:20.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:29:20.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:29:20.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.958231 5210 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:29:20.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.964696 5210 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:29:20.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:20.965132 5210 kserver.cc:163] Server-wide thread pool size limit: 3276
03:29:21.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.049971 5210 rpc_server.cc:307] RPC server started. Bound to: 127.4.252.253:42165
03:29:21.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.050061 5283 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.252.253:42165 every 8 connection(s)
03:29:21.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.052724 5210 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data/info.pb
03:29:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.055507 5107 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu as pid 5210
03:29:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.056010 5107 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal/instance
03:29:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.060254 5284 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:29:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.252.252@KRBTEST.COM; defaulting to no policy
03:29:21.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:29:21.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.252.252@KRBTEST.COM" created.
03:29:21.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.081866 5284 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } has no permanent_uuid. Determining permanent_uuid...
03:29:21.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:21 dist-test-slave-qv09 krb5kdc[5120](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220560, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.253@KRBTEST.COM for kudu/127.4.252.254@KRBTEST.COM
03:29:21.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:29:21.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.252.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.252.keytab.
03:29:21.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.252.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.252.keytab.
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.116828 5107 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/wal
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/logs
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data/info.pb
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:29:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:29:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.252.252:42773
03:29:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.252.252
03:29:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.252.keytab
03:29:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.252.252
03:29:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:29:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:29:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:29:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.252.212:33377
03:29:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773
03:29:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42037/jwks.json
03:29:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:29:21.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.126080 5284 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } has no permanent_uuid. Determining permanent_uuid...
03:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:21 dist-test-slave-qv09 krb5kdc[5120](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220560, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.253@KRBTEST.COM for kudu/127.4.252.253@KRBTEST.COM
03:29:21.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.157662 5284 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } has no permanent_uuid. Determining permanent_uuid...
03:29:21.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.160355 5235 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.165009 5284 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.214578 5284 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } attempt: 1
03:29:21.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.218991 5284 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:21.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.220189 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } attempt: 7
03:29:21.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:21 dist-test-slave-qv09 krb5kdc[5120](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220558, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.254@KRBTEST.COM for kudu/127.4.252.253@KRBTEST.COM
03:29:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.249334 5203 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } has no permanent_uuid. Determining permanent_uuid...
03:29:21.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.255440 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.287510 5284 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } attempt: 2
03:29:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.291448 5284 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:21.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.313927 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } attempt: 1
03:29:21.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.318794 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:21.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.382879 5284 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } attempt: 3
03:29:21.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.387171 5284 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.400210 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } attempt: 2
03:29:21.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.403910 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:21.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.480559 5291 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:29:21.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.481104 5291 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:29:21.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.481658 5291 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:29:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.489310 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } attempt: 3
03:29:21.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.493539 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.517795 5291 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:29:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.518023 5291 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:29:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.518276 5291 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:29:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.518464 5291 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:29:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.518648 5291 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:29:21.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.518841 5291 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:29:21.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.518982 5291 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42037/jwks.json
03:29:21.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.530633 5284 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } attempt: 4
03:29:21.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.534428 5284 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:21.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.559337 5291 master_runner.cc:386] Master server non-default flags:
03:29:21.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:21.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.252.212:33377
03:29:21.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:21.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:21.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/wal
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.252.keytab
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.252.252
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.252.252:42773
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42037/jwks.json
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data/info.pb
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.252.252
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/logs
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:29:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.560484 5291 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:29:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.561458 5291 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:29:21.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.580029 5303 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:21.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.634029 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } attempt: 4
03:29:21.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.638607 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:21.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.833897 5284 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } attempt: 5
03:29:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.838567 5284 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:21.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.922295 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } attempt: 5
03:29:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.926393 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:22.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.386044 5284 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } attempt: 6
03:29:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:22.393414 5284 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:22.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.479008 5203 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } attempt: 6
03:29:22.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:22.486755 5203 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.252.252:42773: Network error: Client connection negotiation failed: client connection to 127.4.252.252:42773: connect: Connection refused (error 111)
03:29:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.580022 5304 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:21.584308 5291 server_base.cc:1034] running on GCE node
03:29:22.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:21.581425 5306 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:22.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.881778 5291 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:29:22.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.884953 5291 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:29:22.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.886406 5291 hybrid_clock.cc:648] HybridClock initialized: now 1790220562886365 us; error 49 us; skew 500 ppm
03:29:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.887410 5291 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:29:22.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:22 dist-test-slave-qv09 krb5kdc[5120](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220562, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:29:22.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.894095 5291 init.cc:377] Logged in from keytab as kudu/127.4.252.252@KRBTEST.COM (short username kudu)
03:29:22.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.900538 5291 webserver.cc:460] Webserver started at http://127.4.252.252:35091/ using document root <none> and password file <none>
03:29:22.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.901423 5291 fs_manager.cc:359] Metadata directory not provided
03:29:22.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.901604 5291 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:29:22.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.901983 5291 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:29:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.907382 5291 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data/instance:
03:29:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a34302522f9747d5851601b367551ba1"
03:29:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:22 on dist-test-slave-qv09"
03:29:22.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.908447 5291 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/wal/instance:
03:29:22.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a34302522f9747d5851601b367551ba1"
03:29:22.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:22 on dist-test-slave-qv09"
03:29:22.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.918018 5291 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.004s sys 0.004s
03:29:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.924012 5318 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:29:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.925212 5291 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
03:29:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.925513 5291 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data,/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/wal
03:29:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a34302522f9747d5851601b367551ba1"
03:29:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:22 on dist-test-slave-qv09"
03:29:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.925813 5291 fs_report.cc:383] FS layout report
03:29:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:29:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/wal
03:29:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/wal
03:29:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data/data
03:29:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:29:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:29:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:29:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:29:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:29:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:29:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:29:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:29:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:29:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:29:22.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:29:22.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.973342 5291 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:29:22.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.980398 5291 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:29:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:22.980916 5291 kserver.cc:163] Server-wide thread pool size limit: 3276
03:29:23.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.068493 5291 rpc_server.cc:307] RPC server started. Bound to: 127.4.252.252:42773
03:29:23.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.068571 5369 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.252.252:42773 every 8 connection(s)
03:29:23.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.071588 5291 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data/info.pb
03:29:23.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.078378 5370 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:29:23.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.079919 5107 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu as pid 5291
03:29:23.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.080274 5107 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/wal/instance
03:29:23.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.093276 5370 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } has no permanent_uuid. Determining permanent_uuid...
03:29:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:23 dist-test-slave-qv09 krb5kdc[5120](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220562, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.252@KRBTEST.COM for kudu/127.4.252.254@KRBTEST.COM
03:29:23.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:23 dist-test-slave-qv09 krb5kdc[5120](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220556, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.252.252@KRBTEST.COM
03:29:23.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.132442 5370 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } has no permanent_uuid. Determining permanent_uuid...
03:29:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:23 dist-test-slave-qv09 krb5kdc[5120](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220556, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.252.253@KRBTEST.COM
03:29:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:23 dist-test-slave-qv09 krb5kdc[5120](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220562, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.252@KRBTEST.COM for kudu/127.4.252.253@KRBTEST.COM
03:29:23.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.169915 5370 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } has no permanent_uuid. Determining permanent_uuid...
03:29:23.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:23 dist-test-slave-qv09 krb5kdc[5120](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220562, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.252@KRBTEST.COM for kudu/127.4.252.252@KRBTEST.COM
03:29:23.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:23 dist-test-slave-qv09 krb5kdc[5120](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220556, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.252.254@KRBTEST.COM
03:29:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.201122 5370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: Bootstrap starting.
03:29:23.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.202268 5107 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
03:29:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.207069 5370 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: Neither blocks nor log segments found. Creating new log.
03:29:23.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.208786 5370 log.cc:824] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: Log is configured to *not* fsync() on all Append() calls
03:29:23.210 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:29:23.211 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
03:29:23.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.213605 5370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: No bootstrap required, opened a new log
03:29:23.221 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
03:29:23.222 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
03:29:23.225 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.252.254"
port: 34807
}
bound_http_address {
host: "127.4.252.254"
port: 32999
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.252.253"
port: 42165
}
bound_http_address {
host: "127.4.252.253"
port: 43345
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.252.252"
port: 42773
}
bound_http_address {
host: "127.4.252.252"
port: 35091
}
}
}
03:29:23.226 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:29:23.226 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:29:23.226 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:29:23.228 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:29:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.239483 5370 raft_consensus.cc:348] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } }
03:29:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.240144 5370 raft_consensus.cc:374] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:29:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.240460 5370 raft_consensus.cc:729] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a34302522f9747d5851601b367551ba1, State: Initialized, Role: FOLLOWER
03:29:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.241372 5370 consensus_queue.cc:260] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } }
03:29:23.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.244436 5381 sys_catalog.cc:455] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } } }
03:29:23.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.245317 5381 sys_catalog.cc:458] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [sys.catalog]: This master's current role is: FOLLOWER
03:29:23.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.246206 5370 sys_catalog.cc:564] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [sys.catalog]: configured and running, proceeding with master startup.
03:29:23.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:23 dist-test-slave-qv09 krb5kdc[5120](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220563, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:29:23.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:29:23.263 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:29:23.266 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5cc
03:29:23.272 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:29:23.273 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5cc
03:29:23.275 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:29:23.279 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
03:29:23.282 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:29:23.282 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:29:23.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.298974 5393 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:23.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.299381 5393 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:23.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:23 dist-test-slave-qv09 krb5kdc[5120](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220563, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:29:23.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:29:23.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.282492 5291 master_runner.cc:186] Error getting master registration for 127.4.252.254:34807: OK, instance_id { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" instance_seqno: 1790220558762004 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:29:23.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.319734 5291 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:29:23.324 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:29:23.324 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5cc
03:29:23.326 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:29:23.326 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.252.254:34807
03:29:23.327 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:29:23.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.328508 5107 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu with pid 5133
03:29:23.359 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:29:23.359 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.252.253:42165
03:29:23.360 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:29:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.360806 5107 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu with pid 5210
03:29:23.387 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:29:23.387 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.252.252:42773
03:29:23.388 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:29:23.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.388621 5107 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu with pid 5291
03:29:23.413 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:29:23.413 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.252.254:34807
03:29:23.415 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.416850 5107 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/logs
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data/info.pb
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.252.254:34807
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.252.254
03:29:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32999
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.254.keytab
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.252.254
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.252.212:33377
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42037/jwks.json
03:29:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:29:23.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.739593 5399 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:29:23.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.740284 5399 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:29:23.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.741001 5399 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:29:23.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.772692 5399 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:29:23.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.773046 5399 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:29:23.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.773355 5399 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:29:23.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.773607 5399 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:29:23.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.773854 5399 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:29:23.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.774183 5399 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:29:23.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.774358 5399 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42037/jwks.json
03:29:23.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.813170 5399 master_runner.cc:386] Master server non-default flags:
03:29:23.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:23.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.252.212:33377
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.254.keytab
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.252.254
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.252.254:34807
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42037/jwks.json
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:29:23.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data/info.pb
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.252.254
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32999
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/logs
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.814389 5399 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:29:23.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:23.815456 5399 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:29:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.837878 5405 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:25.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.838774 5406 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:25.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:23.838730 5408 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:25.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.126034 5407 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1286 milliseconds
03:29:25.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.126119 5399 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:29:25.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.127381 5399 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:29:25.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.130409 5399 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:29:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.131840 5399 hybrid_clock.cc:648] HybridClock initialized: now 1790220565131807 us; error 50 us; skew 500 ppm
03:29:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.132608 5399 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:29:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:25 dist-test-slave-qv09 krb5kdc[5120](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220565, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:29:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.154793 5399 init.cc:377] Logged in from keytab as kudu/127.4.252.254@KRBTEST.COM (short username kudu)
03:29:25.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.160277 5399 webserver.cc:460] Webserver started at http://127.4.252.254:32999/ using document root <none> and password file <none>
03:29:25.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.161255 5399 fs_manager.cc:359] Metadata directory not provided
03:29:25.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.161469 5399 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:29:25.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.170377 5399 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
03:29:25.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.174997 5419 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:29:25.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.176234 5399 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.003s
03:29:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.176612 5399 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data,/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal
03:29:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f755225b6a754d71b231fcbebbe70e37"
03:29:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:18 on dist-test-slave-qv09"
03:29:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.179040 5399 fs_report.cc:383] FS layout report
03:29:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:29:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal
03:29:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal
03:29:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data/data
03:29:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:29:25.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:29:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:29:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:29:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:29:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:29:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:29:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:29:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:29:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:29:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:29:25.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.224574 5399 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:29:25.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.230479 5399 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:29:25.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.230999 5399 kserver.cc:163] Server-wide thread pool size limit: 3276
03:29:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.308550 5399 rpc_server.cc:307] RPC server started. Bound to: 127.4.252.254:34807
03:29:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.308827 5471 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.252.254:34807 every 8 connection(s)
03:29:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.311544 5399 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/data/info.pb
03:29:25.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.318500 5107 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu as pid 5399
03:29:25.319 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:29:25.319 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.252.253:42165
03:29:25.319 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.320883 5107 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/logs
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data/info.pb
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:29:25.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:29:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:29:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.252.253:42165
03:29:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.252.253
03:29:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43345
03:29:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.253.keytab
03:29:25.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.252.253
03:29:25.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:29:25.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:29:25.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:29:25.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.252.212:33377
03:29:25.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:25.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:25.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773
03:29:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42037/jwks.json
03:29:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:29:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.322489 5472 sys_catalog.cc:263] Verifying existing consensus state
03:29:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260924 03:29:25.323244 5472 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:29:25.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.323673 5399 master.cc:537] Master@127.4.252.254:34807 shutting down...
03:29:25.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.390715 5399 master.cc:552] Master@127.4.252.254:34807 shutdown complete.
03:29:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:29:25.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.658429 5474 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:29:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.659021 5474 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:29:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.659757 5474 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:29:25.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.692596 5474 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:29:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.692832 5474 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:29:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.693089 5474 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:29:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.693360 5474 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:29:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.693675 5474 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:29:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.693982 5474 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:29:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.694263 5474 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42037/jwks.json
03:29:25.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.740343 5474 master_runner.cc:386] Master server non-default flags:
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.252.212:33377
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.253.keytab
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.252.253
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.252.253:42165
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42037/jwks.json
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data/info.pb
03:29:25.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.252.253
03:29:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43345
03:29:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:29:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:29:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:29:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/logs
03:29:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:29:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:29:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:29:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:29:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:29:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:29:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:29:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:29:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:29:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.741608 5474 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:29:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.742578 5474 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:29:25.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.767932 5481 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.769228 5482 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:25.769374 5484 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:25.771406 5474 server_base.cc:1034] running on GCE node
03:29:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.032366 5474 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:29:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.035192 5474 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:29:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.036697 5474 hybrid_clock.cc:648] HybridClock initialized: now 1790220567036668 us; error 80 us; skew 500 ppm
03:29:27.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.037709 5474 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:29:27.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:27 dist-test-slave-qv09 krb5kdc[5120](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220567, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:29:27.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.043681 5474 init.cc:377] Logged in from keytab as kudu/127.4.252.253@KRBTEST.COM (short username kudu)
03:29:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.049443 5474 webserver.cc:460] Webserver started at http://127.4.252.253:43345/ using document root <none> and password file <none>
03:29:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.050366 5474 fs_manager.cc:359] Metadata directory not provided
03:29:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.050598 5474 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:29:27.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.058472 5474 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
03:29:27.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.062830 5492 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:29:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.064261 5474 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
03:29:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.064644 5474 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data,/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal
03:29:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a2f82d9c20a94427b4f40a49cc87a0c0"
03:29:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:20 on dist-test-slave-qv09"
03:29:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.067178 5474 fs_report.cc:383] FS layout report
03:29:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:29:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal
03:29:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal
03:29:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data/data
03:29:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:29:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:29:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:29:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:29:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:29:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:29:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:29:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:29:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:29:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:29:27.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:29:27.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.102803 5474 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:29:27.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.109598 5474 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:29:27.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.110193 5474 kserver.cc:163] Server-wide thread pool size limit: 3276
03:29:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.187863 5474 rpc_server.cc:307] RPC server started. Bound to: 127.4.252.253:42165
03:29:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.187999 5543 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.252.253:42165 every 8 connection(s)
03:29:27.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.190891 5474 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/data/info.pb
03:29:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.194128 5107 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu as pid 5474
03:29:27.195 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:29:27.196 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.252.252:42773
03:29:27.196 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.197963 5107 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/wal
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/logs
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data/info.pb
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:29:27.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.252.252:42773
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.252.252
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35091
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.252.keytab
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.252.252
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.252.212:33377
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42037/jwks.json
03:29:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:29:27.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.203572 5544 sys_catalog.cc:263] Verifying existing consensus state
03:29:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260924 03:29:27.204695 5544 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:29:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.205179 5474 master.cc:537] Master@127.4.252.253:42165 shutting down...
03:29:27.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.270839 5474 master.cc:552] Master@127.4.252.253:42165 shutdown complete.
03:29:27.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:29:27.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:27.524479 5546 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:29:27.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:27.525022 5546 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:29:27.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:27.525581 5546 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:29:27.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:27.557469 5546 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:29:27.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:27.557703 5546 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:29:27.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:27.557955 5546 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:29:27.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:27.558156 5546 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:29:27.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:27.558343 5546 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:29:27.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:27.558527 5546 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:29:27.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:27.558661 5546 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42037/jwks.json
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.597606 5546 master_runner.cc:386] Master server non-default flags:
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.252.212:33377
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/wal
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/krb5kdc/kudu_127.4.252.252.keytab
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.252.252
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.252.252:42773
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42037/jwks.json
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data/info.pb
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.252.252
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35091
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/logs
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:29:27.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:29:27.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:29:27.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:29:27.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.598724 5546 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:29:27.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.599828 5546 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:29:27.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:27.616752 5555 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:28.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:28.648923 4533 debug-util.cc:398] Leaking SignalData structure 0x7b080009d000 after lost signal to thread 4468
03:29:28.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:28.649595 4533 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7c20 after lost signal to thread 4536
03:29:28.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:27.617789 5552 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:28.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:27.619208 5546 server_base.cc:1034] running on GCE node
03:29:28.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:27.616816 5553 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:29:28.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.881975 5546 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:29:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.884665 5546 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:29:28.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.886154 5546 hybrid_clock.cc:648] HybridClock initialized: now 1790220568886092 us; error 68 us; skew 500 ppm
03:29:28.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.887033 5546 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:29:28.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:28 dist-test-slave-qv09 krb5kdc[5120](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220568, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:29:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.893113 5546 init.cc:377] Logged in from keytab as kudu/127.4.252.252@KRBTEST.COM (short username kudu)
03:29:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.898453 5546 webserver.cc:460] Webserver started at http://127.4.252.252:35091/ using document root <none> and password file <none>
03:29:28.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.899387 5546 fs_manager.cc:359] Metadata directory not provided
03:29:28.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.899592 5546 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:29:28.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.908624 5546 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.007s
03:29:28.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.913980 5564 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:29:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.915346 5546 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.915639 5546 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data,/tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/wal
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a34302522f9747d5851601b367551ba1"
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:29:22 on dist-test-slave-qv09"
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.917994 5546 fs_report.cc:383] FS layout report
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/wal
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/wal
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data/data
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:29:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:29:28.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.978031 5546 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:29:28.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.984609 5546 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:29:28.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:28.985159 5546 kserver.cc:163] Server-wide thread pool size limit: 3276
03:29:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.059475 5546 rpc_server.cc:307] RPC server started. Bound to: 127.4.252.252:42773
03:29:29.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.059572 5615 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.252.252:42773 every 8 connection(s)
03:29:29.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.062422 5546 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskK6vj3Y/test-tmp/mini-kudu-cluster6732895667286482018/master-2/data/info.pb
03:29:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.066962 5107 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu as pid 5546
03:29:29.067 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:29:29.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.077520 5616 sys_catalog.cc:263] Verifying existing consensus state
03:29:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.085700 5616 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: Bootstrap starting.
03:29:29.117 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:29.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.124312 5616 log.cc:824] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: Log is configured to *not* fsync() on all Append() calls
03:29:29.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.129343 5616 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:29:29.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.130141 5616 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: Bootstrap complete.
03:29:29.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.162247 5616 raft_consensus.cc:348] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } }
03:29:29.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.165834 5616 raft_consensus.cc:374] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:29:29.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.166152 5616 raft_consensus.cc:729] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a34302522f9747d5851601b367551ba1, State: Initialized, Role: FOLLOWER
03:29:29.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.167163 5616 consensus_queue.cc:260] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } }
03:29:29.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.172080 5620 sys_catalog.cc:455] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } } }
03:29:29.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.173152 5620 sys_catalog.cc:458] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [sys.catalog]: This master's current role is: FOLLOWER
03:29:29.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.173889 5616 sys_catalog.cc:564] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [sys.catalog]: configured and running, proceeding with master startup.
03:29:29.175 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:29.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.176563 5620 raft_consensus.cc:482] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:29:29.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.177148 5620 raft_consensus.cc:504] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } }
03:29:29.189 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.252.252:42773(127.4.252.252:42773)] connecting to peer
03:29:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.181309 5620 leader_election.cc:288] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807), a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165)
03:29:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:29.187494 5565 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111)
03:29:29.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:29.203404 5565 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807): Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111)
03:29:29.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:29.205015 5566 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165): Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:29.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.205427 5566 leader_election.cc:302] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a34302522f9747d5851601b367551ba1; no voters: a2f82d9c20a94427b4f40a49cc87a0c0, f755225b6a754d71b231fcbebbe70e37
03:29:29.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.206276 5620 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:29:29.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:29.230862 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:29.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:29.231216 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:29.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:29.241050 5570 tablet.cc:2369] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:29:29.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.225641 5546 master_runner.cc:186] Error getting master registration for 127.4.252.254:34807: Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111),
03:29:29.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:29.253901 5546 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:29:29.261 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.262 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.263 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.263 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.264 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.267 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.4.252.252:42773(127.4.252.252:42773)] Successfully connected to peer
03:29:29.346 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
03:29:29.355 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.252.252
03:29:29.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:29 dist-test-slave-qv09 krb5kdc[5120](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220563, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.252.252@KRBTEST.COM
03:29:29.632 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x65e1ef9e, L:/127.0.0.1:39048 - R:/127.4.252.252:42773] using SASL/GSSAPI
03:29:29.651 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:29.673 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:29.674 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:29.677 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.677 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.678 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.678 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.679 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.686 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:29.692 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:29.693 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:29.695 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.695 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.695 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.698 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:29.712 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:29.714 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:29.716 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.716 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.716 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.721 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.721 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.722 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.725 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:29.752 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:29.753 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:29.756 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.756 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.756 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.760 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.760 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.761 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.762 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:29.792 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:29.793 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:29.794 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.795 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.795 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.799 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.799 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.800 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.801 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:29.812 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:29.813 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:29.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.819 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.820 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.820 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.821 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.822 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:29.952 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:29.954 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:29.955 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.955 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.955 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:29.957 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.957 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.957 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:29.960 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:30.132 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:30.133 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:30.134 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:30.134 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:30.135 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:30.139 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:30.139 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:30.141 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:30.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:30.234647 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:30.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:30.234997 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:30.352 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:30.354 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:30.355 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:30.355 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:30.356 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:30.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:30.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:30.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:30.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:30.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:30.749164 5639 raft_consensus.cc:482] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:29:30.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:30.749719 5639 raft_consensus.cc:504] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } }
03:29:30.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:30.752485 5639 leader_election.cc:288] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807), a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165)
03:29:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:30.765684 5565 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807): Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111)
03:29:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:30.766201 5566 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165): Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:30.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:30.766616 5566 leader_election.cc:302] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a34302522f9747d5851601b367551ba1; no voters: a2f82d9c20a94427b4f40a49cc87a0c0, f755225b6a754d71b231fcbebbe70e37
03:29:30.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:30.767335 5639 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:29:30.972 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:30.973 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:30.975 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:30.975 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:30.975 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:30.975 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:30.976 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:30.976 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:30.980 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:31.238472 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:31.238801 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:31.652 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:31.654 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:31.655 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:31.656 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:31.656 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:31.657 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:31.657 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:31.657 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:31.660 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:31.932 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:31.933 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:31.934 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:31.935 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:31.935 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:31.937 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:31.937 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:31.937 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:31.939 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:32.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:32.241555 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:32.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:32.241832 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:33.152 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:33.154 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:33.155 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:33.156 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:33.156 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:33.156 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:33.156 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:33.156 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:33.160 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:33.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:33.244573 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:33.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:33.244866 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:33.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:33.981843 5648 raft_consensus.cc:482] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:29:33.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:33.982209 5648 raft_consensus.cc:504] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } }
03:29:33.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:33.983886 5648 leader_election.cc:288] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807), a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165)
03:29:33.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:33.989766 5566 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165): Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:33.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:33.990319 5565 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807): Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111)
03:29:33.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:33.990695 5565 leader_election.cc:302] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a34302522f9747d5851601b367551ba1; no voters: a2f82d9c20a94427b4f40a49cc87a0c0, f755225b6a754d71b231fcbebbe70e37
03:29:33.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:33.991430 5648 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:29:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:34.247850 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:34.248143 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:34.256055 5546 master_runner.cc:186] Error getting master registration for 127.4.252.254:34807: Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111),
03:29:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:34.256440 5546 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:29:35.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:35.250909 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:35.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:35.251232 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:36.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:36.254128 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:36.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:36.254444 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:36.832 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:36.834 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:36.835 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:36.835 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:36.836 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:36.836 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:36.836 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:36.837 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:36.840 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:37.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:37.257375 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:37.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:37.257668 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:38.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:38.261343 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:38.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:38.261744 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:38.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:38.759654 5657 raft_consensus.cc:482] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:29:38.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:38.760131 5657 raft_consensus.cc:504] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } }
03:29:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:38.761992 5657 leader_election.cc:288] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807), a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165)
03:29:38.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:38.764824 5565 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111) [suppressed 5 similar messages]
03:29:38.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:38.767818 5566 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165): Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:38.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:38.768256 5565 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807): Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111)
03:29:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:38.768708 5565 leader_election.cc:302] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a34302522f9747d5851601b367551ba1; no voters: a2f82d9c20a94427b4f40a49cc87a0c0, f755225b6a754d71b231fcbebbe70e37
03:29:38.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:38.769467 5657 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:29:38.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:38 dist-test-slave-qv09 krb5kdc[5120](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220578, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:38.896976 5560 init.cc:295] Successfully reacquired a new kerberos TGT
03:29:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:39.258818 5546 master_runner.cc:186] Error getting master registration for 127.4.252.254:34807: Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111),
03:29:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:39.259284 5546 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:29:39.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:39.266405 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:39.266758 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:39.292 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:39.293 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:39.295 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:39.295 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:39.295 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:39.296 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:39.296 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:39.297 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:39.299 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:39.592 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:39.594 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:39.595 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:39.595 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:39.596 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:39.596 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:39.596 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:39.597 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:39.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:40.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:40.269994 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:40.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:40.270318 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:41.273435 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:41.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:41.273674 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:42.172 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:42.174 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:42.175 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:42.175 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:42.176 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:42.176 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:42.177 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:42.177 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:42.179 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:42.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:42.276576 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:42.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:42.276832 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:42.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:42.901335 5664 raft_consensus.cc:482] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:29:42.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:42.901682 5664 raft_consensus.cc:504] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } }
03:29:42.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:42.903079 5664 leader_election.cc:288] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807), a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165)
03:29:42.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:42.907462 5565 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807): Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111)
03:29:42.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:42.907985 5566 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165): Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:42.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:42.908375 5566 leader_election.cc:302] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a34302522f9747d5851601b367551ba1; no voters: a2f82d9c20a94427b4f40a49cc87a0c0, f755225b6a754d71b231fcbebbe70e37
03:29:42.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:42.909108 5664 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:29:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:43.279910 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:43.280197 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:43.492 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:43.494 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:43.494 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:43.494 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:43.495 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:43.497 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:43.497 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:43.497 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:43.499 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:44.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:44.261992 5546 master_runner.cc:186] Error getting master registration for 127.4.252.254:34807: Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111),
03:29:44.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:44.262367 5546 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:29:44.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:44.282958 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:44.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:44.283238 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:45.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:45.286315 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:45.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:45.286566 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:45.552 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:45.554 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:45.555 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:45.555 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:45.555 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:45.556 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:45.556 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:45.556 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:45.558 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:46.289471 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:46.289736 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:46.352 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:46.353 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:46.353 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:46.353 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:46.353 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:46.354 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:46.354 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:46.354 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:46.357 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:46.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:46.464336 5673 raft_consensus.cc:482] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:29:46.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:46.464828 5673 raft_consensus.cc:504] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } }
03:29:46.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:46.466396 5673 leader_election.cc:288] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807), a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165)
03:29:46.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:46.468686 5565 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111) [suppressed 3 similar messages]
03:29:46.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:46.471347 5566 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165): Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:46.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:46.471714 5565 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807): Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111)
03:29:46.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:46.471987 5565 leader_election.cc:302] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a34302522f9747d5851601b367551ba1; no voters: a2f82d9c20a94427b4f40a49cc87a0c0, f755225b6a754d71b231fcbebbe70e37
03:29:46.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:46.472504 5673 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:29:47.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:47.292533 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:47.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:47.292798 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:48.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:48.296413 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:48.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:48.296751 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:48.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:48.875905 5679 raft_consensus.cc:482] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:29:48.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:48.876442 5679 raft_consensus.cc:504] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } }
03:29:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:48.878489 5679 leader_election.cc:288] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807), a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165)
03:29:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:48.883581 5565 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807): Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111)
03:29:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:48.884256 5566 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165): Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:48.884600 5566 leader_election.cc:302] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a34302522f9747d5851601b367551ba1; no voters: a2f82d9c20a94427b4f40a49cc87a0c0, f755225b6a754d71b231fcbebbe70e37
03:29:48.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:48.885218 5679 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:29:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:29:48 dist-test-slave-qv09 krb5kdc[5120](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790220588, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.252.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:29:48.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:48.899611 5560 init.cc:295] Successfully reacquired a new kerberos TGT
03:29:49.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:49.265241 5546 master_runner.cc:186] Error getting master registration for 127.4.252.254:34807: Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111),
03:29:49.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:49.265774 5546 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:29:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:49.299662 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:49.299919 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:49.933 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:49.933 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:49.934 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:49.934 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:49.934 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:49.935 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:49.935 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:49.936 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:49.938 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:50.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:50.303470 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:50.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:50.303727 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:51.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:51.306689 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:51.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:51.306947 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:52.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:52.309901 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:52.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:52.310176 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:52.533 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:52.534 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:52.534 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:52.534 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:52.535 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:52.535 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:52.536 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:52.536 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:52.538 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:52.613 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:52.613 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:52.614 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:52.614 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:52.614 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:52.615 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:52.615 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:52.616 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:52.618 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:52.883922 5688 raft_consensus.cc:482] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:29:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:52.884291 5688 raft_consensus.cc:504] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f755225b6a754d71b231fcbebbe70e37" member_type: VOTER last_known_addr { host: "127.4.252.254" port: 34807 } } peers { permanent_uuid: "a2f82d9c20a94427b4f40a49cc87a0c0" member_type: VOTER last_known_addr { host: "127.4.252.253" port: 42165 } } peers { permanent_uuid: "a34302522f9747d5851601b367551ba1" member_type: VOTER last_known_addr { host: "127.4.252.252" port: 42773 } }
03:29:52.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:52.885819 5688 leader_election.cc:288] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807), a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165)
03:29:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:52.887845 5565 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111) [suppressed 3 similar messages]
03:29:52.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:52.890609 5565 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f755225b6a754d71b231fcbebbe70e37 (127.4.252.254:34807): Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111)
03:29:52.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:52.891003 5566 leader_election.cc:334] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a2f82d9c20a94427b4f40a49cc87a0c0 (127.4.252.253:42165): Network error: Client connection negotiation failed: client connection to 127.4.252.253:42165: connect: Connection refused (error 111)
03:29:52.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:52.891345 5566 leader_election.cc:302] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a34302522f9747d5851601b367551ba1; no voters: a2f82d9c20a94427b4f40a49cc87a0c0, f755225b6a754d71b231fcbebbe70e37
03:29:52.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:52.891865 5688 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:29:53.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:53.313093 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:53.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:53.313351 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:54.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:54.268452 5546 master_runner.cc:186] Error getting master registration for 127.4.252.254:34807: Network error: Client connection negotiation failed: client connection to 127.4.252.254:34807: connect: Connection refused (error 111),
03:29:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:54.268847 5546 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:29:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:54.316143 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:54.316423 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:55.312 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:55.313 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:55.314 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:55.314 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:55.314 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:55.315 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:55.315 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:55.315 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:55.317 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:55.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:55.319041 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:55.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:55.319343 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:56.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:56.322036 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:56.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:56.322293 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:57.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:57.325119 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:57.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:57.325381 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:58.328970 5636 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a34302522f9747d5851601b367551ba1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:29:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:29:58.329382 5636 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:29:58.412 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.254:34807(127.4.252.254:34807)] connecting to peer
03:29:58.413 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.252.253:42165(127.4.252.253:42165)] connecting to peer
03:29:58.413 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:58.413 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.252.254:34807(127.4.252.254:34807)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:58.414 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.254:34807: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807
03:29:58.414 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:58.414 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.253:42165(127.4.252.253:42165)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:58.415 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.252.253:42165: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
03:29:58.417 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773; will retry
03:29:58.420 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29347), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [188ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [187ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [572ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [577ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [602ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [603ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [610ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [611ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [623ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [625ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [626ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [646ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [649ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [650ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [685ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [687ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [720ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [724ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [726ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [744ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [747ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [880ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [882ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [884ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [885ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1060ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1065ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1067ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1068ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1277ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1283ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1285ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1286ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1897ms] refreshing cache from master, [1897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1901ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1901ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1904ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1905ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2577ms] refreshing cache from master, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [2580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [2581ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [2582ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2585ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [2585ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2857ms] refreshing cache from master, [2857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [2858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [2859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [2860ms] too many traces: truncated at 100 traces, deferred=Deferred@193057050(state=PENDING, result=null, callback=(continuation of Deferred@344024018 after retry RPC after error@189181498) -> (continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853), errback=(continuation of Deferred@344024018 after retry RPC after error@189181498) -> (continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853)))
03:29:58.425 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.252.254:34807(127.4.252.254:34807)] disconnecting while in state TERMINATED
03:29:58.427 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.252.252:42773(127.4.252.252:42773)] disconnecting while in state READY
03:29:58.427 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.252.253:42165(127.4.252.253:42165)] disconnecting while in state TERMINATED
03:29:58.432 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:29:58.434 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.4.252.252:42773(127.4.252.252:42773)] handling channelInactive
03:29:58.435 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.252.252:42773(127.4.252.252:42773)] cleaning up while in state READY due to: connection closed
03:29:58.435 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:29:58.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:29:58.444773 5107 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu with pid 5546
03:29:59.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T03:29:59Z chronyd exiting
03:30:00.688 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:30:00.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:30:00.690184 4382 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu with pid 4667
03:30:00.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:30:00.716499 4382 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu with pid 4825
03:30:00.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:30:00.745842 4382 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu with pid 4966
03:30:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:30:00.767059 4382 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu with pid 4394
03:30:00.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:30:00.795089 4382 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu with pid 4464
03:30:00.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:30:00.825519 4382 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskK6vj3Y/build/tsan/bin/kudu with pid 4539
03:30:01.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T03:30:01Z chronyd exiting
03:30:03.031 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29347), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [188ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [187ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [572ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [577ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [602ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [603ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [610ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [611ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [623ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [625ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [626ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [646ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [649ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [650ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [685ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [687ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [720ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [724ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [726ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [744ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [747ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [880ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [882ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [884ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [885ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1060ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1065ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1067ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1068ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1277ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1283ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1285ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1286ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1897ms] refreshing cache from master, [1897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1901ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1901ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1904ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1905ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2577ms] refreshing cache from master, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [2580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [2581ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [2582ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2585ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [2585ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2857ms] refreshing cache from master, [2857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [2858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [2859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [2860ms] too many traces: truncated at 100 traces, deferred=Deferred@193057050(state=PENDING, result=null, callback=(continuation of Deferred@344024018 after retry RPC after error@189181498) -> (continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853), errback=(continuation of Deferred@344024018 after retry RPC after error@189181498) -> (continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.252.253:42165
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29347), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [188ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [187ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [572ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [577ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [602ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [603ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [610ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [611ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [623ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [625ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [626ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [646ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [649ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [650ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [685ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [687ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [720ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [724ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [726ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [744ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [747ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [880ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [882ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [884ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [885ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1060ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1065ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1067ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1068ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1277ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1283ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1285ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1286ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1897ms] refreshing cache from master, [1897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1901ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1901ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1904ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1905ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2577ms] refreshing cache from master, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [2580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [2581ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [2582ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2585ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [2585ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2857ms] refreshing cache from master, [2857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [2858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [2859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [2860ms] too many traces: truncated at 100 traces, deferred=Deferred@193057050(state=PENDING, result=null, callback=(continuation of Deferred@344024018 after retry RPC after error@189181498) -> (continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853), errback=(continuation of Deferred@344024018 after retry RPC after error@189181498) -> (continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.252.253:42165
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
03:30:03.396 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
03:30:03.396 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29347), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [188ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [187ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [572ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [577ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [602ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [603ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [610ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [611ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [623ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [625ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [626ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [646ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [649ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [650ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [685ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [687ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [720ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [724ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [726ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [744ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [747ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [880ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [882ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [884ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [885ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1060ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1065ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1067ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1068ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1277ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1283ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1285ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1286ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1897ms] refreshing cache from master, [1897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1901ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1901ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1904ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1905ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2577ms] refreshing cache from master, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [2580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [2581ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [2582ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2585ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [2585ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2857ms] refreshing cache from master, [2857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [2858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [2859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [2860ms] too many traces: truncated at 100 traces, deferred=Deferred@193057050(state=PENDING, result=null, callback=(continuation of Deferred@344024018 after retry RPC after error@189181498) -> (continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853), errback=(continuation of Deferred@344024018 after retry RPC after error@189181498) -> (continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.252.253:42165
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29347), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [114ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [188ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [187ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [572ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [577ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [602ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [603ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [610ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [611ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [620ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [623ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [625ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [626ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [641ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [646ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [649ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [650ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [681ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [685ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [687ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [720ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [724ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [726ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [744ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [745ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [747ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [880ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [882ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [884ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [885ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1060ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1065ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1067ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1068ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1277ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1279ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1283ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1285ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1286ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1897ms] refreshing cache from master, [1897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [1898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [1900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [1901ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [1901ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [1904ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [1905ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2577ms] refreshing cache from master, [2577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [2578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [2580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [2581ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.254:34807: Network error: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807, [2582ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.253:42165: Network error: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2585ms] Sub RPC ConnectToMaster: received response from server master-127.4.252.252:42773: OK, [2585ms] delaying RPC due to: Service unavailable: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165, [2857ms] refreshing cache from master, [2857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.254:34807, [2858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.253:42165, [2859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.252.252:42773, [2860ms] too many traces: truncated at 100 traces, deferred=Deferred@193057050(state=PENDING, result=null, callback=(continuation of Deferred@344024018 after retry RPC after error@189181498) -> (continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853), errback=(continuation of Deferred@344024018 after retry RPC after error@189181498) -> (continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.252.254:34807,127.4.252.253:42165,127.4.252.252:42773) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.254:34807(127.4.252.254:34807): Connection refused: /127.4.252.254:34807,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.252.253:42165(127.4.252.253:42165): Connection refused: /127.4.252.253:42165
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.252.253:42165
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)