19:14:01.100 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
19:14:01.113 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskuIdNT5/test-tmp
19:14:01.116 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
19:14:01.119 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskuIdNT5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
19:14:01.338 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
19:14:01.544 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:14:01.545 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
19:14:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:01.550207 8679 mini_oidc.cc:152] Starting JWKS server
19:14:01.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:01.554612 8679 webserver.cc:460] Webserver started at http://127.0.0.1:35317/ using document root <none> and password file <none>
19:14:01.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:01.555069 8679 mini_oidc.cc:189] Starting OIDC Discovery server
19:14:01.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:01.557652 8679 webserver.cc:460] Webserver started at http://127.0.0.1:45067/ using document root <none> and password file <none>
19:14:01.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T19:14:01Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
19:14:01.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T19:14:01Z Disabled control of system clock
19:14:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:01.605564 8679 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/wal
19:14:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/data
19:14:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:14:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:14:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:14:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/logs
19:14:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/data/info.pb
19:14:01.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:01.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:14:01.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:14:01.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:14:01.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:14:01.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:01.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:01.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:01.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:14:01.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:14:01.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:01.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:01.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.121.254:35539
19:14:01.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.121.254
19:14:01.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:01.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.121.212:38333
19:14:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.121.254:35539,127.8.121.253:42963,127.8.121.252:45901
19:14:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35317/jwks.json with env {}
19:14:01.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:01.910424 8691 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:14:01.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:01.911075 8691 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:14:01.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:01.911664 8691 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:14:01.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:01.942973 8691 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:14:01.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:01.943265 8691 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:14:01.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:01.943565 8691 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:14:01.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:01.943821 8691 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:14:01.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:01.944058 8691 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:14:01.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:01.944336 8691 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35317/jwks.json
19:14:01.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:01.985261 8691 master_runner.cc:386] Master server non-default flags:
19:14:01.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:01.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.121.212:38333
19:14:01.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:01.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:01.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/data
19:14:01.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/wal
19:14:01.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.121.254:35539,127.8.121.253:42963,127.8.121.252:45901
19:14:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.121.254:35539
19:14:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:14:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35317/jwks.json
19:14:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/data/info.pb
19:14:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.121.254
19:14:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:14:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:01.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:14:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:14:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/logs
19:14:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:14:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:14:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:14:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:14:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:14:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:14:01.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 18:30:25 UTC on ccycloud.quasar-sjmjgc.root.comops.site
19:14:01.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457678
19:14:01.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:14:01.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:01.986513 8691 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:14:01.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:01.988233 8691 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:14:02.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:02.006433 8698 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:03.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:02.007169 8697 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:03.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:02.008492 8691 server_base.cc:1034] running on GCE node
19:14:03.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:02.006546 8700 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.196593 8691 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:14:03.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.199256 8691 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:14:03.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.200584 8691 hybrid_clock.cc:648] HybridClock initialized: now 1781982843200545 us; error 69 us; skew 500 ppm
19:14:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.201400 8691 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:14:03.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.211670 8691 webserver.cc:460] Webserver started at http://127.8.121.254:40541/ using document root <none> and password file <none>
19:14:03.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.212590 8691 fs_manager.cc:359] Metadata directory not provided
19:14:03.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.212769 8691 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:14:03.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.213143 8691 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:14:03.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.218387 8691 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/data/instance:
19:14:03.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "19bbe45d786040bd8ada0c374e216973"
19:14:03.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:03 on dist-test-slave-qhn0"
19:14:03.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.219483 8691 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/wal/instance:
19:14:03.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "19bbe45d786040bd8ada0c374e216973"
19:14:03.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:03 on dist-test-slave-qhn0"
19:14:03.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.226586 8691 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
19:14:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.232321 8707 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:14:03.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.233529 8691 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
19:14:03.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.233824 8691 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/data,/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/wal
19:14:03.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "19bbe45d786040bd8ada0c374e216973"
19:14:03.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:03 on dist-test-slave-qhn0"
19:14:03.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.234203 8691 fs_report.cc:383] FS layout report
19:14:03.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:14:03.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/wal
19:14:03.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/wal
19:14:03.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/data/data
19:14:03.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:14:03.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:14:03.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:14:03.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:14:03.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:14:03.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:14:03.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:14:03.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:14:03.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:14:03.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:14:03.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:14:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.270161 8691 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:14:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.271569 8691 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:14:03.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.271968 8691 kserver.cc:163] Server-wide thread pool size limit: 3276
19:14:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.346190 8691 rpc_server.cc:307] RPC server started. Bound to: 127.8.121.254:35539
19:14:03.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.346282 8758 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.121.254:35539 every 8 connection(s)
19:14:03.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.348822 8691 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/data/info.pb
19:14:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.353909 8759 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:14:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.357236 8679 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu as pid 8691
19:14:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.357614 8679 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-0/wal/instance
19:14:03.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.361752 8679 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:03.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:03.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/wal
19:14:03.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/data
19:14:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:14:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:14:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:14:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/logs
19:14:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/data/info.pb
19:14:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:14:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:14:03.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:14:03.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:14:03.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:03.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:03.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:03.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:14:03.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:14:03.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:03.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.121.253:42963
19:14:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.121.253
19:14:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.121.212:38333
19:14:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.121.254:35539,127.8.121.253:42963,127.8.121.252:45901
19:14:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35317/jwks.json with env {}
19:14:03.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.367473 8759 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } has no permanent_uuid. Determining permanent_uuid...
19:14:03.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.401459 8759 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } has no permanent_uuid. Determining permanent_uuid...
19:14:03.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.403847 8711 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.121.253:42963: connect: Connection refused (error 111)
19:14:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.407837 8759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.253:42963: Network error: Client connection negotiation failed: client connection to 127.8.121.253:42963: connect: Connection refused (error 111)
19:14:03.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.457331 8759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } attempt: 1
19:14:03.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.462479 8759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.253:42963: Network error: Client connection negotiation failed: client connection to 127.8.121.253:42963: connect: Connection refused (error 111)
19:14:03.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.530897 8759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } attempt: 2
19:14:03.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.534965 8759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.253:42963: Network error: Client connection negotiation failed: client connection to 127.8.121.253:42963: connect: Connection refused (error 111)
19:14:03.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.626359 8759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } attempt: 3
19:14:03.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.630476 8759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.253:42963: Network error: Client connection negotiation failed: client connection to 127.8.121.253:42963: connect: Connection refused (error 111)
19:14:03.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.672919 8761 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:14:03.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.673614 8761 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:14:03.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.674295 8761 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:14:03.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.704981 8761 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:14:03.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.705356 8761 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:14:03.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.705669 8761 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:14:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.706027 8761 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:14:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.706354 8761 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:14:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.706624 8761 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35317/jwks.json
19:14:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.741571 8761 master_runner.cc:386] Master server non-default flags:
19:14:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.121.212:38333
19:14:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/data
19:14:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/wal
19:14:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.121.254:35539,127.8.121.253:42963,127.8.121.252:45901
19:14:03.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:03.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:03.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:03.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.121.253:42963
19:14:03.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:03.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:14:03.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35317/jwks.json
19:14:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/data/info.pb
19:14:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.121.253
19:14:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:14:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:14:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:14:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/logs
19:14:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:14:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:14:03.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:14:03.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:14:03.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:14:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:14:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 18:30:25 UTC on ccycloud.quasar-sjmjgc.root.comops.site
19:14:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457678
19:14:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:14:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.743135 8761 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:14:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.745381 8761 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:14:03.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.764442 8770 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.773912 8759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } attempt: 4
19:14:03.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.777514 8759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.253:42963: Network error: Client connection negotiation failed: client connection to 127.8.121.253:42963: connect: Connection refused (error 111)
19:14:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.076933 8759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } attempt: 5
19:14:04.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:04.081415 8759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.253:42963: Network error: Client connection negotiation failed: client connection to 127.8.121.253:42963: connect: Connection refused (error 111)
19:14:04.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.628876 8759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } attempt: 6
19:14:04.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:04.636999 8759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.253:42963: Network error: Client connection negotiation failed: client connection to 127.8.121.253:42963: connect: Connection refused (error 111)
19:14:04.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:03.767257 8761 server_base.cc:1034] running on GCE node
19:14:04.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.765228 8771 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:04.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:03.764604 8773 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:04.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.963971 8761 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:14:04.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.967388 8761 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:14:04.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.968837 8761 hybrid_clock.cc:648] HybridClock initialized: now 1781982844968793 us; error 64 us; skew 500 ppm
19:14:04.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.969950 8761 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:14:04.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.977128 8761 webserver.cc:460] Webserver started at http://127.8.121.253:43269/ using document root <none> and password file <none>
19:14:04.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.978173 8761 fs_manager.cc:359] Metadata directory not provided
19:14:04.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.978365 8761 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:14:04.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.978751 8761 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:14:04.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.983721 8761 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/data/instance:
19:14:04.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8daff24f1bc64098a74411098e56326a"
19:14:04.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:04 on dist-test-slave-qhn0"
19:14:04.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.984867 8761 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/wal/instance:
19:14:04.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8daff24f1bc64098a74411098e56326a"
19:14:04.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:04 on dist-test-slave-qhn0"
19:14:04.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.992384 8761 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
19:14:04.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.997681 8782 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:14:04.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.998903 8761 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
19:14:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.999272 8761 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/data,/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/wal
19:14:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8daff24f1bc64098a74411098e56326a"
19:14:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:04 on dist-test-slave-qhn0"
19:14:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:04.999972 8761 fs_report.cc:383] FS layout report
19:14:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:14:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/wal
19:14:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/wal
19:14:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/data/data
19:14:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:14:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:14:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:14:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:14:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:14:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:14:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:14:05.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:14:05.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:14:05.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:14:05.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:14:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.048846 8761 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:14:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.050338 8761 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:14:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.050822 8761 kserver.cc:163] Server-wide thread pool size limit: 3276
19:14:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.124317 8761 rpc_server.cc:307] RPC server started. Bound to: 127.8.121.253:42963
19:14:05.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.124401 8833 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.121.253:42963 every 8 connection(s)
19:14:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.127631 8761 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/data/info.pb
19:14:05.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.133066 8834 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:14:05.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.134513 8679 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu as pid 8761
19:14:05.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.134923 8679 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-1/wal/instance
19:14:05.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.137317 8679 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:05.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:05.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/wal
19:14:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/data
19:14:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:14:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:14:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:14:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/logs
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/data/info.pb
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:14:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.121.252:45901
19:14:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.121.252
19:14:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.121.212:38333
19:14:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:05.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:05.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.121.254:35539,127.8.121.253:42963,127.8.121.252:45901
19:14:05.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:05.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35317/jwks.json with env {}
19:14:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.145414 8834 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } has no permanent_uuid. Determining permanent_uuid...
19:14:05.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.171303 8834 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } has no permanent_uuid. Determining permanent_uuid...
19:14:05.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.189064 8834 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } has no permanent_uuid. Determining permanent_uuid...
19:14:05.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.191426 8786 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:05.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.195825 8834 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.252:45901: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:05.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.245275 8834 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } attempt: 1
19:14:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.250945 8834 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.252:45901: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:05.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.319326 8834 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } attempt: 2
19:14:05.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.323252 8834 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.252:45901: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:05.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.414659 8834 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } attempt: 3
19:14:05.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.419147 8834 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.252:45901: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.449499 8836 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:14:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.450167 8836 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:14:05.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.450800 8836 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:14:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.498950 8836 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:14:05.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.499295 8836 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:14:05.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.499641 8836 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:14:05.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.499976 8836 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:14:05.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.500308 8836 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:14:05.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.500541 8836 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35317/jwks.json
19:14:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.535077 8836 master_runner.cc:386] Master server non-default flags:
19:14:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.121.212:38333
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/data
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/wal
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.121.254:35539,127.8.121.253:42963,127.8.121.252:45901
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.121.252:45901
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35317/jwks.json
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/data/info.pb
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.121.252
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/logs
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:14:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 18:30:25 UTC on ccycloud.quasar-sjmjgc.root.comops.site
19:14:05.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457678
19:14:05.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:14:05.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.536715 8836 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:14:05.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.538828 8836 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:14:05.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.556790 8847 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.562532 8834 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } attempt: 4
19:14:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.565912 8834 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.252:45901: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.697418 8759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } attempt: 7
19:14:05.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.710879 8759 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } has no permanent_uuid. Determining permanent_uuid...
19:14:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.717877 8759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.252:45901: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.776332 8759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } attempt: 1
19:14:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.780902 8759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.252:45901: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:05.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.862278 8759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } attempt: 2
19:14:05.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.865356 8834 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } attempt: 5
19:14:05.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.867600 8759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.252:45901: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:05.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.869438 8834 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.252:45901: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.952986 8759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } attempt: 3
19:14:05.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.957173 8759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.252:45901: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:06.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.097558 8759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } attempt: 4
19:14:06.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:06.101949 8759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.252:45901: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:06.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.385321 8759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } attempt: 5
19:14:06.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:06.389637 8759 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.252:45901: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.417006 8834 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } attempt: 6
19:14:06.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:06.424378 8834 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.121.252:45901: Network error: Client connection negotiation failed: client connection to 127.8.121.252:45901: connect: Connection refused (error 111)
19:14:06.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.557775 8849 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:06.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:05.557921 8846 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:06.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:05.558408 8836 server_base.cc:1034] running on GCE node
19:14:06.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.743976 8836 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:14:06.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.747157 8836 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:14:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.748621 8836 hybrid_clock.cc:648] HybridClock initialized: now 1781982846748582 us; error 61 us; skew 500 ppm
19:14:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.749408 8836 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:14:06.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.756645 8836 webserver.cc:460] Webserver started at http://127.8.121.252:46731/ using document root <none> and password file <none>
19:14:06.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.757572 8836 fs_manager.cc:359] Metadata directory not provided
19:14:06.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.757742 8836 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:14:06.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.758183 8836 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:14:06.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.763027 8836 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/data/instance:
19:14:06.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bef4e0d28a184dd79c1bdbe51622a998"
19:14:06.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:06 on dist-test-slave-qhn0"
19:14:06.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.764072 8836 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/wal/instance:
19:14:06.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bef4e0d28a184dd79c1bdbe51622a998"
19:14:06.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:06 on dist-test-slave-qhn0"
19:14:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.772081 8836 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
19:14:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.777493 8861 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:14:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.779057 8836 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
19:14:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.779383 8836 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/data,/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/wal
19:14:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bef4e0d28a184dd79c1bdbe51622a998"
19:14:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:06 on dist-test-slave-qhn0"
19:14:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.779768 8836 fs_report.cc:383] FS layout report
19:14:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:14:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/wal
19:14:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/wal
19:14:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/data/data
19:14:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:14:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:14:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:14:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:14:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:14:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:14:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:14:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:14:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:14:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:14:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:14:06.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.816260 8836 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:14:06.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.817744 8836 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:14:06.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.818241 8836 kserver.cc:163] Server-wide thread pool size limit: 3276
19:14:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.892434 8836 rpc_server.cc:307] RPC server started. Bound to: 127.8.121.252:45901
19:14:06.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.892551 8912 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.121.252:45901 every 8 connection(s)
19:14:06.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.895310 8836 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/data/info.pb
19:14:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.901122 8913 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:14:06.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.904232 8679 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu as pid 8836
19:14:06.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.904628 8679 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/master-2/wal/instance
19:14:06.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.916143 8913 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } has no permanent_uuid. Determining permanent_uuid...
19:14:06.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.942121 8759 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } attempt: 6
19:14:06.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.962365 8913 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } has no permanent_uuid. Determining permanent_uuid...
19:14:06.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.975828 8913 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } has no permanent_uuid. Determining permanent_uuid...
19:14:06.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.977473 8759 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973: Bootstrap starting.
19:14:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.984457 8759 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973: Neither blocks nor log segments found. Creating new log.
19:14:06.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.986496 8759 log.cc:824] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973: Log is configured to *not* fsync() on all Append() calls
19:14:06.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.991636 8759 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973: No bootstrap required, opened a new log
19:14:06.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:06.996060 8913 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998: Bootstrap starting.
19:14:07.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.003831 8913 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998: Neither blocks nor log segments found. Creating new log.
19:14:07.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.007174 8913 log.cc:824] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998: Log is configured to *not* fsync() on all Append() calls
19:14:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.015518 8913 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998: No bootstrap required, opened a new log
19:14:07.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.021366 8759 raft_consensus.cc:348] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } }
19:14:07.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.022414 8759 raft_consensus.cc:374] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:14:07.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.022667 8759 raft_consensus.cc:729] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 19bbe45d786040bd8ada0c374e216973, State: Initialized, Role: FOLLOWER
19:14:07.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.023571 8759 consensus_queue.cc:260] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } }
19:14:07.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.026928 8925 sys_catalog.cc:455] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } } }
19:14:07.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.028859 8925 sys_catalog.cc:458] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [sys.catalog]: This master's current role is: FOLLOWER
19:14:07.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.028884 8759 sys_catalog.cc:564] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [sys.catalog]: configured and running, proceeding with master startup.
19:14:07.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:07.030642 8713 tablet.cc:2369] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:14:07.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.042232 8913 raft_consensus.cc:348] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } }
19:14:07.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.042845 8913 raft_consensus.cc:374] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:14:07.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.043108 8913 raft_consensus.cc:729] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bef4e0d28a184dd79c1bdbe51622a998, State: Initialized, Role: FOLLOWER
19:14:07.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.044128 8913 consensus_queue.cc:260] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } }
19:14:07.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.051568 8934 sys_catalog.cc:455] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } } }
19:14:07.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.052580 8934 sys_catalog.cc:458] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [sys.catalog]: This master's current role is: FOLLOWER
19:14:07.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.053421 8913 sys_catalog.cc:564] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [sys.catalog]: configured and running, proceeding with master startup.
19:14:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:07.071491 8938 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:07.072781 8938 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:07.073427 8867 tablet.cc:2369] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:14:07.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.064519 8691 master_runner.cc:186] Error getting master registration for 127.8.121.253:42963: OK, instance_id { permanent_uuid: "8daff24f1bc64098a74411098e56326a" instance_seqno: 1781982845104560 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:14:07.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.101222 8691 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:14:07.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:07.109136 8949 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:07.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:07.109689 8949 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:07.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.103245 8836 master_runner.cc:186] Error getting master registration for 127.8.121.253:42963: OK, instance_id { permanent_uuid: "8daff24f1bc64098a74411098e56326a" instance_seqno: 1781982845104560 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:14:07.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.138041 8836 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:14:07.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.264254 8934 raft_consensus.cc:482] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:14:07.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.264787 8934 raft_consensus.cc:504] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } }
19:14:07.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.267275 8934 leader_election.cc:288] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 19bbe45d786040bd8ada0c374e216973 (127.8.121.254:35539), 8daff24f1bc64098a74411098e56326a (127.8.121.253:42963)
19:14:07.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.268299 8734 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bef4e0d28a184dd79c1bdbe51622a998" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "19bbe45d786040bd8ada0c374e216973" is_pre_election: true
19:14:07.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.269187 8734 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bef4e0d28a184dd79c1bdbe51622a998 in term 0.
19:14:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.270816 8862 leader_election.cc:302] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 19bbe45d786040bd8ada0c374e216973, bef4e0d28a184dd79c1bdbe51622a998; no voters:
19:14:07.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.271668 8934 raft_consensus.cc:2781] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [term 0 FOLLOWER]: Leader pre-election won for term 1
19:14:07.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.271986 8934 raft_consensus.cc:482] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
19:14:07.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.272281 8934 raft_consensus.cc:3037] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [term 0 FOLLOWER]: Advancing to term 1
19:14:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.279052 8934 raft_consensus.cc:504] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } }
19:14:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.281625 8734 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bef4e0d28a184dd79c1bdbe51622a998" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "19bbe45d786040bd8ada0c374e216973"
19:14:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.282259 8734 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [term 0 FOLLOWER]: Advancing to term 1
19:14:07.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.282398 8934 leader_election.cc:288] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [CANDIDATE]: Term 1 election: Requested vote from peers 19bbe45d786040bd8ada0c374e216973 (127.8.121.254:35539), 8daff24f1bc64098a74411098e56326a (127.8.121.253:42963)
19:14:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.295272 8734 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bef4e0d28a184dd79c1bdbe51622a998 in term 1.
19:14:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.298508 8862 leader_election.cc:302] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 19bbe45d786040bd8ada0c374e216973, bef4e0d28a184dd79c1bdbe51622a998; no voters:
19:14:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.302429 8934 raft_consensus.cc:2781] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [term 1 FOLLOWER]: Leader election won for term 1
19:14:07.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.322778 8934 raft_consensus.cc:686] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [term 1 LEADER]: Becoming Leader. State: Replica: bef4e0d28a184dd79c1bdbe51622a998, State: Running, Role: LEADER
19:14:07.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.324162 8934 consensus_queue.cc:237] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } }
19:14:07.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.349340 8951 sys_catalog.cc:455] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bef4e0d28a184dd79c1bdbe51622a998. Latest consensus state: current_term: 1 leader_uuid: "bef4e0d28a184dd79c1bdbe51622a998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } } }
19:14:07.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.351125 8951 sys_catalog.cc:458] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [sys.catalog]: This master's current role is: LEADER
19:14:07.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.368685 8953 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
19:14:07.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.384091 8953 catalog_manager.cc:1484] Initializing Kudu cluster ID...
19:14:07.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.420188 8734 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [term 1 FOLLOWER]: Refusing update from remote peer bef4e0d28a184dd79c1bdbe51622a998: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:14:07.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.427101 8934 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:14:07.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.462661 8954 mvcc.cc:205] Tried to move back new op lower bound from 7299001743014694912 to 7299001742650163200. Current Snapshot: MvccSnapshot[applied={T|T < 7299001743014694912}]
19:14:07.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.463896 8951 sys_catalog.cc:455] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bef4e0d28a184dd79c1bdbe51622a998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } } }
19:14:07.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.464627 8951 sys_catalog.cc:458] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [sys.catalog]: This master's current role is: LEADER
19:14:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.465734 8953 catalog_manager.cc:1347] Generated new cluster ID: 0bf662b1f0524306b36a6c8cfe063f3a
19:14:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.466038 8953 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
19:14:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.470556 8925 sys_catalog.cc:455] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bef4e0d28a184dd79c1bdbe51622a998. Latest consensus state: current_term: 1 leader_uuid: "bef4e0d28a184dd79c1bdbe51622a998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } } }
19:14:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.471230 8925 sys_catalog.cc:458] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [sys.catalog]: This master's current role is: FOLLOWER
19:14:07.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.483080 8925 sys_catalog.cc:455] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bef4e0d28a184dd79c1bdbe51622a998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } } }
19:14:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.484165 8925 sys_catalog.cc:458] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973 [sys.catalog]: This master's current role is: FOLLOWER
19:14:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.485275 8834 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } attempt: 7
19:14:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.525463 8834 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a: Bootstrap starting.
19:14:07.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.537303 8834 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a: Neither blocks nor log segments found. Creating new log.
19:14:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.545624 8834 log.cc:824] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a: Log is configured to *not* fsync() on all Append() calls
19:14:07.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.546182 8953 catalog_manager.cc:1370] Generated new certificate authority record
19:14:07.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.551411 8953 catalog_manager.cc:1504] Loading token signing keys...
19:14:07.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.558676 8834 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a: No bootstrap required, opened a new log
19:14:07.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.624230 8953 catalog_manager.cc:5868] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998: Generated new TSK 0
19:14:07.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.625742 8953 catalog_manager.cc:1514] Initializing in-progress tserver states...
19:14:07.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.634343 8834 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } }
19:14:07.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.635171 8834 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:14:07.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.635439 8834 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8daff24f1bc64098a74411098e56326a, State: Initialized, Role: FOLLOWER
19:14:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.636413 8834 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } }
19:14:07.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.641700 8834 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a [sys.catalog]: configured and running, proceeding with master startup.
19:14:07.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.642230 8807 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a [term 0 FOLLOWER]: Advancing to term 1
19:14:07.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.646179 8808 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bef4e0d28a184dd79c1bdbe51622a998" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8daff24f1bc64098a74411098e56326a"
19:14:07.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.648142 8809 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bef4e0d28a184dd79c1bdbe51622a998" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8daff24f1bc64098a74411098e56326a" is_pre_election: true
19:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.667444 8679 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/wal
19:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/data
19:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/logs
19:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/data/info.pb
19:14:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.121.193:0
19:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.121.193
19:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.121.193
19:14:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:07.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.121.254:35539,127.8.121.253:42963,127.8.121.252:45901
19:14:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.121.212:38333
19:14:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:14:07.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.676630 8807 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a [term 1 FOLLOWER]: Refusing update from remote peer bef4e0d28a184dd79c1bdbe51622a998: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:14:07.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.686100 8934 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
19:14:07.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.640416 8961 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } } }
19:14:07.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.697046 8961 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a [sys.catalog]: This master's current role is: FOLLOWER
19:14:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.782181 8961 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a [sys.catalog]: SysCatalogTable state changed. Reason: New leader bef4e0d28a184dd79c1bdbe51622a998. Latest consensus state: current_term: 1 leader_uuid: "bef4e0d28a184dd79c1bdbe51622a998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } } }
19:14:07.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.782456 8969 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bef4e0d28a184dd79c1bdbe51622a998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } } }
19:14:07.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.785982 8969 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a [sys.catalog]: This master's current role is: FOLLOWER
19:14:07.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.786206 8961 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a [sys.catalog]: This master's current role is: FOLLOWER
19:14:07.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:07.806294 8978 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
19:14:08.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:08.082710 8938 catalog_manager.cc:1259] Loaded cluster ID: 0bf662b1f0524306b36a6c8cfe063f3a
19:14:08.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:08.083540 8938 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973: loading cluster ID for follower catalog manager: success
19:14:08.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:08.094910 8938 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973: acquiring CA information for follower catalog manager: success
19:14:08.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:08.101155 8938 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 19bbe45d786040bd8ada0c374e216973: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:14:08.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:08.136956 8971 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:14:08.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:08.137850 8971 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:14:08.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:08.138759 8971 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:14:08.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:08.192137 8971 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:14:08.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:08.193091 8971 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.121.193
19:14:08.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:08.222908 8934 sys_catalog.cc:455] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bef4e0d28a184dd79c1bdbe51622a998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "19bbe45d786040bd8ada0c374e216973" member_type: VOTER last_known_addr { host: "127.8.121.254" port: 35539 } } peers { permanent_uuid: "8daff24f1bc64098a74411098e56326a" member_type: VOTER last_known_addr { host: "127.8.121.253" port: 42963 } } peers { permanent_uuid: "bef4e0d28a184dd79c1bdbe51622a998" member_type: VOTER last_known_addr { host: "127.8.121.252" port: 45901 } } }
19:14:08.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:08.224051 8934 sys_catalog.cc:458] T 00000000000000000000000000000000 P bef4e0d28a184dd79c1bdbe51622a998 [sys.catalog]: This master's current role is: LEADER
19:14:08.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:08.249291 8971 tablet_server_runner.cc:78] Tablet server non-default flags:
19:14:08.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:08.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.121.212:38333
19:14:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/data
19:14:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/wal
19:14:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.121.193:0
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/data/info.pb
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.121.193
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.121.254:35539,127.8.121.253:42963,127.8.121.252:45901
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.121.193
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/logs
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 18:30:25 UTC on ccycloud.quasar-sjmjgc.root.comops.site
19:14:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457678
19:14:08.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:14:08.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:08.250716 8971 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:14:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:08.252822 8971 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:14:08.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:08.280272 8987 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:08.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:08.825697 8978 catalog_manager.cc:1259] Loaded cluster ID: 0bf662b1f0524306b36a6c8cfe063f3a
19:14:08.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:08.838503 8978 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a: loading cluster ID for follower catalog manager: success
19:14:08.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:08.870674 8978 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a: acquiring CA information for follower catalog manager: success
19:14:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:08.899921 8978 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 8daff24f1bc64098a74411098e56326a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:14:10.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:08.282630 8986 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:10.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:10.565989 8971 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.285s user 0.463s sys 0.903s
19:14:10.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:09.670367 8985 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 8971
19:14:10.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:10.569024 8990 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:10.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:10.569622 8971 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.288s user 0.463s sys 0.904s
19:14:10.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:10.570749 8988 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2285 milliseconds
19:14:10.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.570749 8971 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:14:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.572688 8971 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:14:10.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.578230 8971 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:14:10.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.579881 8971 hybrid_clock.cc:648] HybridClock initialized: now 1781982850579847 us; error 47 us; skew 500 ppm
19:14:10.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.580857 8971 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:14:10.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.592036 8971 webserver.cc:460] Webserver started at http://127.8.121.193:41135/ using document root <none> and password file <none>
19:14:10.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.592999 8971 fs_manager.cc:359] Metadata directory not provided
19:14:10.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.593233 8971 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:14:10.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.593768 8971 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:14:10.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.598793 8971 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/data/instance:
19:14:10.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8291f460c54c49e7ab66aac1edeeeb2b"
19:14:10.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:10 on dist-test-slave-qhn0"
19:14:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.599891 8971 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/wal/instance:
19:14:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8291f460c54c49e7ab66aac1edeeeb2b"
19:14:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:10 on dist-test-slave-qhn0"
19:14:10.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.607590 8971 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
19:14:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.613210 9000 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:14:10.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.614355 8971 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
19:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.614681 8971 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/data,/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/wal
19:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8291f460c54c49e7ab66aac1edeeeb2b"
19:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:10 on dist-test-slave-qhn0"
19:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.615016 8971 fs_report.cc:383] FS layout report
19:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/wal
19:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/wal
19:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/data/data
19:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:14:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:14:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:14:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:14:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:14:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:14:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:14:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:14:10.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.660498 8971 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:14:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.662353 8971 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:14:10.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.662880 8971 kserver.cc:163] Server-wide thread pool size limit: 3276
19:14:10.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.666010 8971 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:14:10.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.671447 8971 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:14:10.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.671736 8971 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:14:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.672020 8971 ts_tablet_manager.cc:610] Registered 0 tablets
19:14:10.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.672216 8971 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:14:10.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.837738 8971 rpc_server.cc:307] RPC server started. Bound to: 127.8.121.193:37515
19:14:10.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.837812 9112 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.121.193:37515 every 8 connection(s)
19:14:10.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.841329 8971 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/data/info.pb
19:14:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.845961 8679 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu as pid 8971
19:14:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.846469 8679 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-0/wal/instance
19:14:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.868587 8679 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/wal
19:14:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/data
19:14:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:14:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:14:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:14:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:10.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/logs
19:14:10.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/data/info.pb
19:14:10.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:10.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:14:10.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:14:10.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:14:10.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.121.194:0
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.121.194
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.121.194
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.121.254:35539,127.8.121.253:42963,127.8.121.252:45901
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.121.212:38333
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:14:10.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.932080 9113 heartbeater.cc:347] Connected to a master server at 127.8.121.252:45901
19:14:10.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.932492 9113 heartbeater.cc:464] Registering TS with master...
19:14:10.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.933706 9113 heartbeater.cc:511] Master 127.8.121.252:45901 requested a full tablet report, sending...
19:14:10.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.937882 8878 ts_manager.cc:194] Registered new tserver with Master: 8291f460c54c49e7ab66aac1edeeeb2b (127.8.121.193:37515)
19:14:10.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.940758 8878 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.121.193:56269
19:14:10.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.947568 9114 heartbeater.cc:347] Connected to a master server at 127.8.121.253:42963
19:14:10.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.947886 9114 heartbeater.cc:464] Registering TS with master...
19:14:10.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.948447 9117 heartbeater.cc:347] Connected to a master server at 127.8.121.254:35539
19:14:10.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.948527 9114 heartbeater.cc:511] Master 127.8.121.253:42963 requested a full tablet report, sending...
19:14:10.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.948760 9117 heartbeater.cc:464] Registering TS with master...
19:14:10.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.949343 9117 heartbeater.cc:511] Master 127.8.121.254:35539 requested a full tablet report, sending...
19:14:10.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.951902 8799 ts_manager.cc:194] Registered new tserver with Master: 8291f460c54c49e7ab66aac1edeeeb2b (127.8.121.193:37515)
19:14:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:10.952564 8724 ts_manager.cc:194] Registered new tserver with Master: 8291f460c54c49e7ab66aac1edeeeb2b (127.8.121.193:37515)
19:14:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:11.235718 9118 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:14:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:11.236172 9118 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:14:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:11.236701 9118 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:14:11.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:11.269901 9118 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:14:11.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:11.270687 9118 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.121.194
19:14:11.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:11.304466 9118 tablet_server_runner.cc:78] Tablet server non-default flags:
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.121.212:38333
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/data
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/wal
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.121.194:0
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/data/info.pb
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.121.194
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.121.254:35539,127.8.121.253:42963,127.8.121.252:45901
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:14:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.121.194
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/logs
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 18:30:25 UTC on ccycloud.quasar-sjmjgc.root.comops.site
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457678
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:11.305576 9118 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:14:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:11.307157 9118 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:14:11.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:11.331847 9129 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:11.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:11.946040 9113 heartbeater.cc:503] Master 127.8.121.252:45901 was elected leader, sending a full tablet report...
19:14:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:11.332587 9130 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:11.338003 9118 server_base.cc:1034] running on GCE node
19:14:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:11.339635 9132 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:12.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.559935 9118 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:14:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.562848 9118 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:14:12.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.564261 9118 hybrid_clock.cc:648] HybridClock initialized: now 1781982852564225 us; error 62 us; skew 500 ppm
19:14:12.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.564962 9118 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:14:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.571519 9118 webserver.cc:460] Webserver started at http://127.8.121.194:36265/ using document root <none> and password file <none>
19:14:12.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.572340 9118 fs_manager.cc:359] Metadata directory not provided
19:14:12.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.572520 9118 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:14:12.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.572928 9118 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:14:12.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.578019 9118 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/data/instance:
19:14:12.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "857b1f26977d428ca3d518b733211473"
19:14:12.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:12 on dist-test-slave-qhn0"
19:14:12.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.579095 9118 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/wal/instance:
19:14:12.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "857b1f26977d428ca3d518b733211473"
19:14:12.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:12 on dist-test-slave-qhn0"
19:14:12.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.585812 9118 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.002s sys 0.005s
19:14:12.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.591164 9143 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:14:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.592293 9118 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
19:14:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.592587 9118 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/data,/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/wal
19:14:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "857b1f26977d428ca3d518b733211473"
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:12 on dist-test-slave-qhn0"
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.592885 9118 fs_report.cc:383] FS layout report
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/wal
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/wal
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/data/data
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:14:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:14:12.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:14:12.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:14:12.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:14:12.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.625475 9118 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:14:12.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.626911 9118 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:14:12.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.627332 9118 kserver.cc:163] Server-wide thread pool size limit: 3276
19:14:12.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.630751 9118 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:14:12.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.636883 9118 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:14:12.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.637173 9118 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:14:12.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.637488 9118 ts_tablet_manager.cc:610] Registered 0 tablets
19:14:12.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.637725 9118 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:14:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.782015 9118 rpc_server.cc:307] RPC server started. Bound to: 127.8.121.194:33393
19:14:12.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.782121 9255 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.121.194:33393 every 8 connection(s)
19:14:12.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.784544 9118 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/data/info.pb
19:14:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.796087 8679 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu as pid 9118
19:14:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.796744 8679 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-1/wal/instance
19:14:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.811863 8679 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/wal
19:14:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/data
19:14:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:14:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:14:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:14:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/logs
19:14:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/data/info.pb
19:14:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.121.195:0
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.121.195
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.121.195
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.121.254:35539,127.8.121.253:42963,127.8.121.252:45901
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.121.212:38333
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:14:12.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.823951 9256 heartbeater.cc:347] Connected to a master server at 127.8.121.252:45901
19:14:12.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.824442 9256 heartbeater.cc:464] Registering TS with master...
19:14:12.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.825618 9256 heartbeater.cc:511] Master 127.8.121.252:45901 requested a full tablet report, sending...
19:14:12.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.828589 8878 ts_manager.cc:194] Registered new tserver with Master: 857b1f26977d428ca3d518b733211473 (127.8.121.194:33393)
19:14:12.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.830770 8878 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.121.194:52411
19:14:12.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.837512 9257 heartbeater.cc:347] Connected to a master server at 127.8.121.253:42963
19:14:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.837880 9257 heartbeater.cc:464] Registering TS with master...
19:14:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.838579 9257 heartbeater.cc:511] Master 127.8.121.253:42963 requested a full tablet report, sending...
19:14:12.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.841521 8799 ts_manager.cc:194] Registered new tserver with Master: 857b1f26977d428ca3d518b733211473 (127.8.121.194:33393)
19:14:12.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.845314 9258 heartbeater.cc:347] Connected to a master server at 127.8.121.254:35539
19:14:12.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.845616 9258 heartbeater.cc:464] Registering TS with master...
19:14:12.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.846225 9258 heartbeater.cc:511] Master 127.8.121.254:35539 requested a full tablet report, sending...
19:14:12.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:12.848256 8724 ts_manager.cc:194] Registered new tserver with Master: 857b1f26977d428ca3d518b733211473 (127.8.121.194:33393)
19:14:13.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:13.127148 9266 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:14:13.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:13.127676 9266 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:14:13.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:13.128391 9266 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:14:13.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:13.160893 9266 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:14:13.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:13.161705 9266 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.121.195
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:13.194999 9266 tablet_server_runner.cc:78] Tablet server non-default flags:
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.121.212:38333
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/data
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/wal
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.121.195:0
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/data/info.pb
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.121.195
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.121.254:35539,127.8.121.253:42963,127.8.121.252:45901
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.121.195
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/logs
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 18:30:25 UTC on ccycloud.quasar-sjmjgc.root.comops.site
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457678
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:14:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:13.196245 9266 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:14:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:13.198303 9266 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:14:13.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:13.219303 9272 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:13.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:13.836404 9256 heartbeater.cc:503] Master 127.8.121.252:45901 was elected leader, sending a full tablet report...
19:14:14.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:13.222776 9266 server_base.cc:1034] running on GCE node
19:14:14.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:13.219892 9275 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:14.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:13.221000 9273 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:14.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.425020 9266 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:14:14.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.427311 9266 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:14:14.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.428684 9266 hybrid_clock.cc:648] HybridClock initialized: now 1781982854428646 us; error 50 us; skew 500 ppm
19:14:14.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.429502 9266 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:14:14.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.436084 9266 webserver.cc:460] Webserver started at http://127.8.121.195:37147/ using document root <none> and password file <none>
19:14:14.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.436940 9266 fs_manager.cc:359] Metadata directory not provided
19:14:14.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.437109 9266 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:14:14.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.437461 9266 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:14:14.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.442181 9266 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/data/instance:
19:14:14.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acf8987a61004350835be9c818053177"
19:14:14.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:14 on dist-test-slave-qhn0"
19:14:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.443207 9266 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/wal/instance:
19:14:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acf8987a61004350835be9c818053177"
19:14:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:14 on dist-test-slave-qhn0"
19:14:14.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.449721 9266 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
19:14:14.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.455333 9283 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:14:14.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.456318 9266 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
19:14:14.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.456593 9266 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/data,/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/wal
19:14:14.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acf8987a61004350835be9c818053177"
19:14:14.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:14 on dist-test-slave-qhn0"
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.456851 9266 fs_report.cc:383] FS layout report
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/wal
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/wal
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/data/data
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:14:14.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:14:14.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.493659 9266 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:14:14.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.495133 9266 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:14:14.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.495546 9266 kserver.cc:163] Server-wide thread pool size limit: 3276
19:14:14.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.498044 9266 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:14:14.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.502362 9266 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:14:14.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.502590 9266 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:14:14.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.502784 9266 ts_tablet_manager.cc:610] Registered 0 tablets
19:14:14.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.502905 9266 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:14:14.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.651180 9266 rpc_server.cc:307] RPC server started. Bound to: 127.8.121.195:35833
19:14:14.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.651237 9395 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.121.195:35833 every 8 connection(s)
19:14:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.653738 9266 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/data/info.pb
19:14:14.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.656842 8679 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu as pid 9266
19:14:14.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.657341 8679 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster7260101564007706792/ts-2/wal/instance
19:14:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.712477 9398 heartbeater.cc:347] Connected to a master server at 127.8.121.254:35539
19:14:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.712705 9396 heartbeater.cc:347] Connected to a master server at 127.8.121.252:45901
19:14:14.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.712993 9398 heartbeater.cc:464] Registering TS with master...
19:14:14.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.713007 9396 heartbeater.cc:464] Registering TS with master...
19:14:14.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.714164 9397 heartbeater.cc:347] Connected to a master server at 127.8.121.253:42963
19:14:14.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.714748 9397 heartbeater.cc:464] Registering TS with master...
19:14:14.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.715046 9398 heartbeater.cc:511] Master 127.8.121.254:35539 requested a full tablet report, sending...
19:14:14.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.715173 9396 heartbeater.cc:511] Master 127.8.121.252:45901 requested a full tablet report, sending...
19:14:14.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.715477 9397 heartbeater.cc:511] Master 127.8.121.253:42963 requested a full tablet report, sending...
19:14:14.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.718271 8798 ts_manager.cc:194] Registered new tserver with Master: acf8987a61004350835be9c818053177 (127.8.121.195:35833)
19:14:14.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.718667 8724 ts_manager.cc:194] Registered new tserver with Master: acf8987a61004350835be9c818053177 (127.8.121.195:35833)
19:14:14.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.720203 8878 ts_manager.cc:194] Registered new tserver with Master: acf8987a61004350835be9c818053177 (127.8.121.195:35833)
19:14:14.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.721964 8878 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.121.195:50457
19:14:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:14.726377 8679 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
19:14:14.737 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:14:14.738 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
19:14:14.750 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.121.254"
port: 35539
}
bound_http_address {
host: "127.8.121.254"
port: 40541
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.121.253"
port: 42963
}
bound_http_address {
host: "127.8.121.253"
port: 43269
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.121.252"
port: 45901
}
bound_http_address {
host: "127.8.121.252"
port: 46731
}
}
}
19:14:14.754 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
19:14:14.756 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.8.121.193"
port: 37515
}
bound_http_address {
host: "127.8.121.193"
port: 41135
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.8.121.194"
port: 33393
}
bound_http_address {
host: "127.8.121.194"
port: 36265
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.8.121.195"
port: 35833
}
bound_http_address {
host: "127.8.121.195"
port: 37147
}
}
}
19:14:14.756 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
19:14:14.946 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
19:14:15.049 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskuIdNT5/test-tmp
19:14:15.050 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
19:14:15.050 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskuIdNT5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
19:14:15.056 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
19:14:15.388 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:14:15.388 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
19:14:15.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:15.390955 9407 mini_oidc.cc:152] Starting JWKS server
19:14:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:15.394836 9407 webserver.cc:460] Webserver started at http://127.0.0.1:36093/ using document root <none> and password file <none>
19:14:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:15.395318 9407 mini_oidc.cc:189] Starting OIDC Discovery server
19:14:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:15.397482 9407 webserver.cc:460] Webserver started at http://127.0.0.1:34225/ using document root <none> and password file <none>
19:14:15.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
19:14:15.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/principal' for realm 'KRBTEST.COM',
19:14:15.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
19:14:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:15 dist-test-slave-qhn0 krb5kdc[9420](info): setting up network...
19:14:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
19:14:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:15 dist-test-slave-qhn0 krb5kdc[9420](info): set up 2 sockets
19:14:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:15 dist-test-slave-qhn0 krb5kdc[9420](info): commencing operation
19:14:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
19:14:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:15.725492 9396 heartbeater.cc:503] Master 127.8.121.252:45901 was elected leader, sending a full tablet report...
19:14:16.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:16.491621 9407 mini_kdc.cc:121] Time spent starting KDC: real 1.069s user 0.005s sys 0.015s
19:14:16.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
19:14:16.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:14:16.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
19:14:16.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
19:14:16.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:14:16.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
19:14:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
19:14:16.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:14:16.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
19:14:16.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:14:16.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/test-user.keytab.
19:14:16.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/test-user.keytab.
19:14:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:16 dist-test-slave-qhn0 krb5kdc[9420](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982856, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:14:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:14:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T19:14:16Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
19:14:16.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T19:14:16Z Disabled control of system clock
19:14:16.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.47.254@KRBTEST.COM; defaulting to no policy
19:14:16.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:14:16.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.47.254@KRBTEST.COM" created.
19:14:16.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:14:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.47.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.254.keytab.
19:14:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.47.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.254.keytab.
19:14:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:16.672295 9407 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal
19:14:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data
19:14:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/logs
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data/info.pb
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.47.254:32789
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.47.254
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.254.keytab
19:14:16.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.47.254
19:14:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:14:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:14:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:14:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.47.212:35579
19:14:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.47.254:32789,127.9.47.253:38325,127.9.47.252:37663
19:14:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36093/jwks.json
19:14:16.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:14:16.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:16.975533 9434 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:14:16.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:16.976182 9434 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:14:16.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:16.976817 9434 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:14:17.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:17.008090 9434 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:14:17.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:17.008315 9434 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:14:17.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:17.008603 9434 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:14:17.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:17.008788 9434 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:14:17.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:17.008955 9434 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:14:17.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:17.009119 9434 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:14:17.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:17.009253 9434 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36093/jwks.json
19:14:17.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:17.046437 9434 master_runner.cc:386] Master server non-default flags:
19:14:17.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:17.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.47.212:35579
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.47.254:32789,127.9.47.253:38325,127.9.47.252:37663
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.254.keytab
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.47.254
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.47.254:32789
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36093/jwks.json
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data/info.pb
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.47.254
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/logs
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 18:30:25 UTC on ccycloud.quasar-sjmjgc.root.comops.site
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457678
19:14:17.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:14:17.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:17.047780 9434 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:14:17.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:17.049269 9434 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:14:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:17.070475 9441 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:18.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:17.070501 9442 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:18.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:17.072376 9434 server_base.cc:1034] running on GCE node
19:14:18.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:17.071643 9444 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:18.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.295339 9434 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:14:18.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.299015 9434 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:14:18.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.300423 9434 hybrid_clock.cc:648] HybridClock initialized: now 1781982858300374 us; error 60 us; skew 500 ppm
19:14:18.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.301270 9434 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:14:18.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:18 dist-test-slave-qhn0 krb5kdc[9420](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982858, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:14:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.310343 9434 init.cc:377] Logged in from keytab as kudu/127.9.47.254@KRBTEST.COM (short username kudu)
19:14:18.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.315860 9434 webserver.cc:460] Webserver started at http://127.9.47.254:43417/ using document root <none> and password file <none>
19:14:18.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.316951 9434 fs_manager.cc:359] Metadata directory not provided
19:14:18.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.317149 9434 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:14:18.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.317620 9434 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:14:18.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.322517 9434 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data/instance:
19:14:18.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86dcbdae9fa24d48b21275380ffbf243"
19:14:18.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:18 on dist-test-slave-qhn0"
19:14:18.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.323602 9434 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal/instance:
19:14:18.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86dcbdae9fa24d48b21275380ffbf243"
19:14:18.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:18 on dist-test-slave-qhn0"
19:14:18.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.331192 9434 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
19:14:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.336408 9453 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:14:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.337534 9434 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.337796 9434 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data,/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86dcbdae9fa24d48b21275380ffbf243"
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:18 on dist-test-slave-qhn0"
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.338152 9434 fs_report.cc:383] FS layout report
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data/data
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:14:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:14:18.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.371495 9434 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:14:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.377768 9434 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:14:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.378223 9434 kserver.cc:163] Server-wide thread pool size limit: 3276
19:14:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.461112 9434 rpc_server.cc:307] RPC server started. Bound to: 127.9.47.254:32789
19:14:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.461184 9504 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.47.254:32789 every 8 connection(s)
19:14:18.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.464293 9434 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data/info.pb
19:14:18.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.469832 9505 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:14:18.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.470623 9407 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu as pid 9434
19:14:18.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.471125 9407 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal/instance
19:14:18.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.483578 9505 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } has no permanent_uuid. Determining permanent_uuid...
19:14:18.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.47.253@KRBTEST.COM; defaulting to no policy
19:14:18.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:14:18.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.47.253@KRBTEST.COM" created.
19:14:18.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:18 dist-test-slave-qhn0 krb5kdc[9420](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982858, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.254@KRBTEST.COM for kudu/127.9.47.254@KRBTEST.COM
19:14:18.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:14:18.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.47.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.253.keytab.
19:14:18.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.47.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.253.keytab.
19:14:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.516275 9407 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal
19:14:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data
19:14:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:14:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:14:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:14:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:18.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/logs
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data/info.pb
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.47.253:38325
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.47.253
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.253.keytab
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.47.253
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.47.212:35579
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.47.254:32789,127.9.47.253:38325,127.9.47.252:37663
19:14:18.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36093/jwks.json
19:14:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:14:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.524392 9505 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } has no permanent_uuid. Determining permanent_uuid...
19:14:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.526621 9455 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.47.253:38325: connect: Connection refused (error 111)
19:14:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.531139 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.253:38325: Network error: Client connection negotiation failed: client connection to 127.9.47.253:38325: connect: Connection refused (error 111)
19:14:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.580703 9505 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } attempt: 1
19:14:18.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.584389 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.253:38325: Network error: Client connection negotiation failed: client connection to 127.9.47.253:38325: connect: Connection refused (error 111)
19:14:18.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.652849 9505 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } attempt: 2
19:14:18.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.657379 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.253:38325: Network error: Client connection negotiation failed: client connection to 127.9.47.253:38325: connect: Connection refused (error 111)
19:14:18.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.749185 9505 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } attempt: 3
19:14:18.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.752733 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.253:38325: Network error: Client connection negotiation failed: client connection to 127.9.47.253:38325: connect: Connection refused (error 111)
19:14:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.837710 9511 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:14:18.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.838443 9511 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:14:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.839304 9511 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:14:18.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.872692 9511 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:14:18.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.872913 9511 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:14:18.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.873238 9511 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:14:18.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.873494 9511 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:14:18.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.873778 9511 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:14:18.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.874060 9511 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:14:18.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.874217 9511 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36093/jwks.json
19:14:18.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.896124 9505 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } attempt: 4
19:14:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.900110 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.253:38325: Network error: Client connection negotiation failed: client connection to 127.9.47.253:38325: connect: Connection refused (error 111)
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.912882 9511 master_runner.cc:386] Master server non-default flags:
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.47.212:35579
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.47.254:32789,127.9.47.253:38325,127.9.47.252:37663
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.253.keytab
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.47.253
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.47.253:38325
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36093/jwks.json
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:14:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data/info.pb
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.47.253
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/logs
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 18:30:25 UTC on ccycloud.quasar-sjmjgc.root.comops.site
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457678
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:14:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.914090 9511 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:14:18.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:18.915023 9511 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:14:18.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.933913 9519 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:19.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:19.199540 9505 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } attempt: 5
19:14:19.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:19.203945 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.253:38325: Network error: Client connection negotiation failed: client connection to 127.9.47.253:38325: connect: Connection refused (error 111)
19:14:19.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:19.751533 9505 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } attempt: 6
19:14:19.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:19.758631 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.253:38325: Network error: Client connection negotiation failed: client connection to 127.9.47.253:38325: connect: Connection refused (error 111)
19:14:20.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.934753 9518 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:20.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:18.934266 9521 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:20.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.166186 9520 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
19:14:20.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.166297 9511 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:14:20.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.170456 9511 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:14:20.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.173287 9511 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:14:20.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.174731 9511 hybrid_clock.cc:648] HybridClock initialized: now 1781982860174660 us; error 60 us; skew 500 ppm
19:14:20.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.175542 9511 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:14:20.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:20 dist-test-slave-qhn0 krb5kdc[9420](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982860, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:14:20.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.181813 9511 init.cc:377] Logged in from keytab as kudu/127.9.47.253@KRBTEST.COM (short username kudu)
19:14:20.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.187819 9511 webserver.cc:460] Webserver started at http://127.9.47.253:38549/ using document root <none> and password file <none>
19:14:20.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.188751 9511 fs_manager.cc:359] Metadata directory not provided
19:14:20.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.188990 9511 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:14:20.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.189499 9511 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:14:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.194209 9511 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data/instance:
19:14:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4c3072111684769b44985016f14a625"
19:14:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:20 on dist-test-slave-qhn0"
19:14:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.195312 9511 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal/instance:
19:14:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4c3072111684769b44985016f14a625"
19:14:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:20 on dist-test-slave-qhn0"
19:14:20.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.202673 9511 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
19:14:20.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.208163 9531 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:14:20.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.209169 9511 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.004s
19:14:20.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.209537 9511 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data,/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal
19:14:20.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4c3072111684769b44985016f14a625"
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:20 on dist-test-slave-qhn0"
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.209910 9511 fs_report.cc:383] FS layout report
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data/data
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:14:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:14:20.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:14:20.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:14:20.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.256042 9511 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:14:20.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.262662 9511 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:14:20.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.263196 9511 kserver.cc:163] Server-wide thread pool size limit: 3276
19:14:20.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.340637 9511 rpc_server.cc:307] RPC server started. Bound to: 127.9.47.253:38325
19:14:20.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.340716 9582 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.47.253:38325 every 8 connection(s)
19:14:20.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.343576 9511 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data/info.pb
19:14:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.347151 9407 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu as pid 9511
19:14:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.347645 9407 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal/instance
19:14:20.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.350998 9583 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:14:20.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.47.252@KRBTEST.COM; defaulting to no policy
19:14:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:14:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.47.252@KRBTEST.COM" created.
19:14:20.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.369053 9583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } has no permanent_uuid. Determining permanent_uuid...
19:14:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:14:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.47.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.252.keytab.
19:14:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.47.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.252.keytab.
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.394588 9407 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/wal
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/logs
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data/info.pb
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.47.252:37663
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.47.252
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.252.keytab
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.47.252
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.47.212:35579
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.47.254:32789,127.9.47.253:38325,127.9.47.252:37663
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36093/jwks.json
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:14:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:20 dist-test-slave-qhn0 krb5kdc[9420](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982860, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.253@KRBTEST.COM for kudu/127.9.47.254@KRBTEST.COM
19:14:20.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.408092 9583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } has no permanent_uuid. Determining permanent_uuid...
19:14:20.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:20 dist-test-slave-qhn0 krb5kdc[9420](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982860, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.253@KRBTEST.COM for kudu/127.9.47.253@KRBTEST.COM
19:14:20.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.435501 9583 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } has no permanent_uuid. Determining permanent_uuid...
19:14:20.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.437999 9532 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:20.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.441740 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:20.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.491262 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } attempt: 1
19:14:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.495046 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:20.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.563424 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } attempt: 2
19:14:20.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.567009 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:20.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.658396 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } attempt: 3
19:14:20.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.662395 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:20.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.713698 9589 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:14:20.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.714247 9589 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:14:20.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.714808 9589 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:14:20.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.747891 9589 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:14:20.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.748108 9589 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:14:20.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.748374 9589 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:14:20.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.748592 9589 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:14:20.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.748776 9589 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:14:20.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.748957 9589 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:14:20.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.749105 9589 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36093/jwks.json
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.788522 9589 master_runner.cc:386] Master server non-default flags:
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.47.212:35579
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/wal
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.47.254:32789,127.9.47.253:38325,127.9.47.252:37663
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.252.keytab
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.47.252
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.47.252:37663
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36093/jwks.json
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:14:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data/info.pb
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.47.252
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/logs
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 18:30:25 UTC on ccycloud.quasar-sjmjgc.root.comops.site
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457678
19:14:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:14:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.790043 9589 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:14:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.791294 9589 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:14:20.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.805768 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } attempt: 4
19:14:20.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.809908 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:20.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.812036 9597 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:20.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.819149 9505 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } attempt: 7
19:14:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:20 dist-test-slave-qhn0 krb5kdc[9420](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982858, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.254@KRBTEST.COM for kudu/127.9.47.253@KRBTEST.COM
19:14:20.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.847869 9505 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } has no permanent_uuid. Determining permanent_uuid...
19:14:20.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.853018 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:20.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.911568 9505 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } attempt: 1
19:14:20.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.915683 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:20.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.997251 9505 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } attempt: 2
19:14:21.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:21.001361 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:21.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:21.086731 9505 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } attempt: 3
19:14:21.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:21.090740 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:21.109280 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } attempt: 5
19:14:21.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:21.114033 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:21.231148 9505 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } attempt: 4
19:14:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:21.236092 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:21.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:21.519536 9505 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } attempt: 5
19:14:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:21.523671 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:21.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:21.661425 9583 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } attempt: 6
19:14:21.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:21.667721 9583 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:22.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.815423 9598 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:22.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:20.816270 9589 server_base.cc:1034] running on GCE node
19:14:22.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:20.815594 9600 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:22.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.056867 9589 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:14:22.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.059391 9589 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:14:22.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.060735 9589 hybrid_clock.cc:648] HybridClock initialized: now 1781982862060695 us; error 46 us; skew 500 ppm
19:14:22.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.061470 9589 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:14:22.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:22 dist-test-slave-qhn0 krb5kdc[9420](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982862, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:14:22.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.066987 9589 init.cc:377] Logged in from keytab as kudu/127.9.47.252@KRBTEST.COM (short username kudu)
19:14:22.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.072674 9589 webserver.cc:460] Webserver started at http://127.9.47.252:33191/ using document root <none> and password file <none>
19:14:22.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.073660 9589 fs_manager.cc:359] Metadata directory not provided
19:14:22.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.073966 9589 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:14:22.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.074534 9589 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:14:22.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.076097 9505 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } attempt: 6
19:14:22.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.081235 9589 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data/instance:
19:14:22.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9502b19ca544311a0507617142f5b14"
19:14:22.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:22 on dist-test-slave-qhn0"
19:14:22.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.082639 9505 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.47.252:37663: Network error: Client connection negotiation failed: client connection to 127.9.47.252:37663: connect: Connection refused (error 111)
19:14:22.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.082695 9589 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/wal/instance:
19:14:22.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9502b19ca544311a0507617142f5b14"
19:14:22.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:22 on dist-test-slave-qhn0"
19:14:22.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.090812 9589 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
19:14:22.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.096848 9617 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:14:22.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.098181 9589 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
19:14:22.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.098547 9589 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data,/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/wal
19:14:22.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9502b19ca544311a0507617142f5b14"
19:14:22.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:22 on dist-test-slave-qhn0"
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.098914 9589 fs_report.cc:383] FS layout report
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/wal
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/wal
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data/data
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:14:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:14:22.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.139143 9589 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:14:22.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.144806 9589 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:14:22.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.145210 9589 kserver.cc:163] Server-wide thread pool size limit: 3276
19:14:22.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.219103 9589 rpc_server.cc:307] RPC server started. Bound to: 127.9.47.252:37663
19:14:22.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.219180 9668 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.47.252:37663 every 8 connection(s)
19:14:22.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.221664 9589 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data/info.pb
19:14:22.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.224921 9407 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu as pid 9589
19:14:22.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.225404 9407 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/wal/instance
19:14:22.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.227882 9669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:14:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.245491 9669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } has no permanent_uuid. Determining permanent_uuid...
19:14:22.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:22 dist-test-slave-qhn0 krb5kdc[9420](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982856, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.47.252@KRBTEST.COM
19:14:22.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:22 dist-test-slave-qhn0 krb5kdc[9420](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982862, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.252@KRBTEST.COM for kudu/127.9.47.254@KRBTEST.COM
19:14:22.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.278450 9669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } has no permanent_uuid. Determining permanent_uuid...
19:14:22.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:22 dist-test-slave-qhn0 krb5kdc[9420](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982862, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.252@KRBTEST.COM for kudu/127.9.47.253@KRBTEST.COM
19:14:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:22 dist-test-slave-qhn0 krb5kdc[9420](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982856, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.47.253@KRBTEST.COM
19:14:22.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.303980 9669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } has no permanent_uuid. Determining permanent_uuid...
19:14:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:22 dist-test-slave-qhn0 krb5kdc[9420](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982862, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.252@KRBTEST.COM for kudu/127.9.47.252@KRBTEST.COM
19:14:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:22 dist-test-slave-qhn0 krb5kdc[9420](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982856, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.47.254@KRBTEST.COM
19:14:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.333727 9669 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: Bootstrap starting.
19:14:22.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.336262 9407 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
19:14:22.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.340058 9669 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: Neither blocks nor log segments found. Creating new log.
19:14:22.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.342415 9669 log.cc:824] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: Log is configured to *not* fsync() on all Append() calls
19:14:22.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:14:22.348 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
19:14:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.351411 9669 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: No bootstrap required, opened a new log
19:14:22.367 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
19:14:22.368 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
19:14:22.375 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.9.47.254"
port: 32789
}
bound_http_address {
host: "127.9.47.254"
port: 43417
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.9.47.253"
port: 38325
}
bound_http_address {
host: "127.9.47.253"
port: 38549
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.9.47.252"
port: 37663
}
bound_http_address {
host: "127.9.47.252"
port: 33191
}
}
}
19:14:22.375 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
19:14:22.375 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:14:22.375 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
19:14:22.377 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
19:14:22.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.383219 9669 raft_consensus.cc:348] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dcbdae9fa24d48b21275380ffbf243" member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } } peers { permanent_uuid: "e4c3072111684769b44985016f14a625" member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } } peers { permanent_uuid: "a9502b19ca544311a0507617142f5b14" member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } }
19:14:22.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.384084 9669 raft_consensus.cc:374] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:14:22.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.384364 9669 raft_consensus.cc:729] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a9502b19ca544311a0507617142f5b14, State: Initialized, Role: FOLLOWER
19:14:22.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.385303 9669 consensus_queue.cc:260] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dcbdae9fa24d48b21275380ffbf243" member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } } peers { permanent_uuid: "e4c3072111684769b44985016f14a625" member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } } peers { permanent_uuid: "a9502b19ca544311a0507617142f5b14" member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } }
19:14:22.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.388476 9679 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dcbdae9fa24d48b21275380ffbf243" member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } } peers { permanent_uuid: "e4c3072111684769b44985016f14a625" member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } } peers { permanent_uuid: "a9502b19ca544311a0507617142f5b14" member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } } }
19:14:22.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.389516 9679 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [sys.catalog]: This master's current role is: FOLLOWER
19:14:22.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.390161 9669 sys_catalog.cc:564] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [sys.catalog]: configured and running, proceeding with master startup.
19:14:22.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.400143 9623 tablet.cc:2369] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:14:22.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:22 dist-test-slave-qhn0 krb5kdc[9420](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982862, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:14:22.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:14:22.414 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:14:22.417 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5cc
19:14:22.425 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:14:22.426 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5cc
19:14:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.426628 9691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:22.427 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:14:22.431 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
19:14:22.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.432927 9691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:22.433 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
19:14:22.434 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
19:14:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.414279 9589 master_runner.cc:186] Error getting master registration for 127.9.47.254:32789: OK, instance_id { permanent_uuid: "86dcbdae9fa24d48b21275380ffbf243" instance_seqno: 1781982858436303 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:14:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.448271 9589 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:14:22.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:22 dist-test-slave-qhn0 krb5kdc[9420](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982862, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:14:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:14:22.470 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:14:22.470 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5cc
19:14:22.472 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:14:22.472 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.47.252:37663
19:14:22.475 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
19:14:22.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.475860 9407 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu with pid 9589
19:14:22.499 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:14:22.499 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.47.254:32789
19:14:22.500 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
19:14:22.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.500480 9407 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu with pid 9434
19:14:22.524 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:14:22.524 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.47.253:38325
19:14:22.524 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
19:14:22.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.524878 9407 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu with pid 9511
19:14:22.544 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:14:22.544 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.47.252:37663
19:14:22.546 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
19:14:22.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.547355 9407 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/wal
19:14:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data
19:14:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:14:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:14:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:14:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/logs
19:14:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data/info.pb
19:14:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:22.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.47.252:37663
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.47.252
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33191
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.252.keytab
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.47.252
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.47.212:35579
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:22.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:22.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:22.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:22.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.47.254:32789,127.9.47.253:38325,127.9.47.252:37663
19:14:22.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:22.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36093/jwks.json
19:14:22.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:14:22.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.852461 9695 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:14:22.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.852981 9695 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:14:22.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.853730 9695 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:14:22.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.886698 9695 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.886901 9695 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.887253 9695 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.887471 9695 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.887673 9695 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.887861 9695 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.887989 9695 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36093/jwks.json
19:14:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.931071 9695 master_runner.cc:386] Master server non-default flags:
19:14:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.47.212:35579
19:14:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data
19:14:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/wal
19:14:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:14:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.47.254:32789,127.9.47.253:38325,127.9.47.252:37663
19:14:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.252.keytab
19:14:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.47.252
19:14:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.47.252:37663
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36093/jwks.json
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data/info.pb
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.47.252
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33191
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/logs
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:14:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:14:22.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:14:22.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:14:22.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 18:30:25 UTC on ccycloud.quasar-sjmjgc.root.comops.site
19:14:22.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457678
19:14:22.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:14:22.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.932199 9695 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:14:22.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.933355 9695 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:14:22.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.951174 9701 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:24.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.951217 9702 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:24.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:22.951210 9704 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:24.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:22.952557 9695 server_base.cc:1034] running on GCE node
19:14:24.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.222157 9695 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:14:24.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.224848 9695 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:14:24.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.226239 9695 hybrid_clock.cc:648] HybridClock initialized: now 1781982864226215 us; error 40 us; skew 500 ppm
19:14:24.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.227092 9695 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:14:24.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:24 dist-test-slave-qhn0 krb5kdc[9420](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982864, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:14:24.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.238296 9695 init.cc:377] Logged in from keytab as kudu/127.9.47.252@KRBTEST.COM (short username kudu)
19:14:24.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.243978 9695 webserver.cc:460] Webserver started at http://127.9.47.252:33191/ using document root <none> and password file <none>
19:14:24.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.244910 9695 fs_manager.cc:359] Metadata directory not provided
19:14:24.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.245157 9695 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:14:24.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.257615 9695 fs_manager.cc:709] Time spent opening directory manager: real 0.008s user 0.005s sys 0.000s
19:14:24.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.263603 9714 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:14:24.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.264956 9695 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
19:14:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.265360 9695 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data,/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/wal
19:14:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9502b19ca544311a0507617142f5b14"
19:14:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:22 on dist-test-slave-qhn0"
19:14:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.269016 9695 fs_report.cc:383] FS layout report
19:14:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:14:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/wal
19:14:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/wal
19:14:24.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data/data
19:14:24.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:14:24.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:14:24.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:14:24.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:14:24.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:14:24.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:14:24.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:14:24.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:14:24.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:14:24.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:14:24.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:14:24.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.313342 9695 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:14:24.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.319944 9695 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:14:24.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.320500 9695 kserver.cc:163] Server-wide thread pool size limit: 3276
19:14:24.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.400975 9695 rpc_server.cc:307] RPC server started. Bound to: 127.9.47.252:37663
19:14:24.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.401074 9765 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.47.252:37663 every 8 connection(s)
19:14:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.403996 9695 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-2/data/info.pb
19:14:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.408957 9407 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu as pid 9695
19:14:24.409 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:14:24.410 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.47.254:32789
19:14:24.410 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.411522 9407 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/logs
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data/info.pb
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.47.254:32789
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.47.254
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43417
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.254.keytab
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.47.254
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.47.212:35579
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.47.254:32789,127.9.47.253:38325,127.9.47.252:37663
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36093/jwks.json
19:14:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:14:24.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.415409 9766 sys_catalog.cc:263] Verifying existing consensus state
19:14:24.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.424248 9766 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: Bootstrap starting.
19:14:24.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.468499 9766 log.cc:824] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: Log is configured to *not* fsync() on all Append() calls
19:14:24.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.473551 9766 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:14:24.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.474407 9766 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: Bootstrap complete.
19:14:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.508366 9766 raft_consensus.cc:348] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dcbdae9fa24d48b21275380ffbf243" member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } } peers { permanent_uuid: "e4c3072111684769b44985016f14a625" member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } } peers { permanent_uuid: "a9502b19ca544311a0507617142f5b14" member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } }
19:14:24.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.511778 9766 raft_consensus.cc:374] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:14:24.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.512091 9766 raft_consensus.cc:729] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a9502b19ca544311a0507617142f5b14, State: Initialized, Role: FOLLOWER
19:14:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.513028 9766 consensus_queue.cc:260] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dcbdae9fa24d48b21275380ffbf243" member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } } peers { permanent_uuid: "e4c3072111684769b44985016f14a625" member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } } peers { permanent_uuid: "a9502b19ca544311a0507617142f5b14" member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } }
19:14:24.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.515762 9770 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dcbdae9fa24d48b21275380ffbf243" member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } } peers { permanent_uuid: "e4c3072111684769b44985016f14a625" member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } } peers { permanent_uuid: "a9502b19ca544311a0507617142f5b14" member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } } }
19:14:24.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.516449 9770 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [sys.catalog]: This master's current role is: FOLLOWER
19:14:24.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.517508 9766 sys_catalog.cc:564] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [sys.catalog]: configured and running, proceeding with master startup.
19:14:24.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.545819 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:24.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.546283 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:24.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.575744 9720 tablet.cc:2369] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:14:24.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.548295 9695 master_runner.cc:186] Error getting master registration for 127.9.47.254:32789: Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111),
19:14:24.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.578896 9695 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:14:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.709148 9770 raft_consensus.cc:482] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:14:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.709743 9770 raft_consensus.cc:504] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dcbdae9fa24d48b21275380ffbf243" member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } } peers { permanent_uuid: "e4c3072111684769b44985016f14a625" member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } } peers { permanent_uuid: "a9502b19ca544311a0507617142f5b14" member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } }
19:14:24.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.715445 9715 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111)
19:14:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.718583 9770 leader_election.cc:288] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86dcbdae9fa24d48b21275380ffbf243 (127.9.47.254:32789), e4c3072111684769b44985016f14a625 (127.9.47.253:38325)
19:14:24.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.722478 9716 leader_election.cc:334] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4c3072111684769b44985016f14a625 (127.9.47.253:38325): Network error: Client connection negotiation failed: client connection to 127.9.47.253:38325: connect: Connection refused (error 111)
19:14:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.724681 9715 leader_election.cc:334] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 86dcbdae9fa24d48b21275380ffbf243 (127.9.47.254:32789): Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111)
19:14:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.725054 9715 leader_election.cc:302] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a9502b19ca544311a0507617142f5b14; no voters: 86dcbdae9fa24d48b21275380ffbf243, e4c3072111684769b44985016f14a625
19:14:24.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.725821 9770 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:14:24.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.810207 9768 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:14:24.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.810765 9768 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:14:24.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.811466 9768 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:14:24.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.851555 9768 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:14:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.851776 9768 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:14:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.852039 9768 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:14:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.852233 9768 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:14:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.852420 9768 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:14:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.852609 9768 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:14:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.852759 9768 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36093/jwks.json
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.897737 9768 master_runner.cc:386] Master server non-default flags:
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.47.212:35579
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.47.254:32789,127.9.47.253:38325,127.9.47.252:37663
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.254.keytab
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.47.254
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.47.254:32789
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:14:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36093/jwks.json
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data/info.pb
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.47.254
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43417
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/logs
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 18:30:25 UTC on ccycloud.quasar-sjmjgc.root.comops.site
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457678
19:14:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:14:24.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.899255 9768 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:14:24.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:24.900558 9768 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:14:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.927105 9792 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:25.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:25.551775 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:25.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:25.552527 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:26.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.330617 9799 raft_consensus.cc:482] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:14:26.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.331418 9799 raft_consensus.cc:504] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dcbdae9fa24d48b21275380ffbf243" member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } } peers { permanent_uuid: "e4c3072111684769b44985016f14a625" member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } } peers { permanent_uuid: "a9502b19ca544311a0507617142f5b14" member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } }
19:14:26.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.343015 9799 leader_election.cc:288] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86dcbdae9fa24d48b21275380ffbf243 (127.9.47.254:32789), e4c3072111684769b44985016f14a625 (127.9.47.253:38325)
19:14:26.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:26.350628 9716 leader_election.cc:334] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4c3072111684769b44985016f14a625 (127.9.47.253:38325): Network error: Client connection negotiation failed: client connection to 127.9.47.253:38325: connect: Connection refused (error 111)
19:14:26.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:26.351255 9715 leader_election.cc:334] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 86dcbdae9fa24d48b21275380ffbf243 (127.9.47.254:32789): Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111)
19:14:26.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.351816 9715 leader_election.cc:302] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a9502b19ca544311a0507617142f5b14; no voters: 86dcbdae9fa24d48b21275380ffbf243, e4c3072111684769b44985016f14a625
19:14:26.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.352892 9799 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:14:26.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:26.557173 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:26.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:26.558418 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:26.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:26.319733 9790 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 9768
19:14:26.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:26.947290 9790 kernel_stack_watchdog.cc:198] Thread 9768 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 387ms:
19:14:26.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
19:14:26.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
19:14:26.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
19:14:26.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
19:14:26.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:26.950302 9768 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.021s user 0.001s sys 0.001s
19:14:26.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:26.950697 9768 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.022s user 0.001s sys 0.001s
19:14:26.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:24.927135 9791 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:26.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:26.954314 9795 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:26.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:26.959936 9793 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2030 milliseconds
19:14:26.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.959939 9768 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:14:26.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.961335 9768 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:14:26.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.964345 9768 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:14:26.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.967955 9768 hybrid_clock.cc:648] HybridClock initialized: now 1781982866967888 us; error 81 us; skew 500 ppm
19:14:26.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.969033 9768 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:14:26.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:26 dist-test-slave-qhn0 krb5kdc[9420](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982866, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:14:26.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.976320 9768 init.cc:377] Logged in from keytab as kudu/127.9.47.254@KRBTEST.COM (short username kudu)
19:14:26.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.982272 9768 webserver.cc:460] Webserver started at http://127.9.47.254:43417/ using document root <none> and password file <none>
19:14:26.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.983273 9768 fs_manager.cc:359] Metadata directory not provided
19:14:26.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.983520 9768 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:14:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.991726 9768 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
19:14:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.996579 9810 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:14:26.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.997799 9768 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
19:14:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:26.998179 9768 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data,/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal
19:14:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86dcbdae9fa24d48b21275380ffbf243"
19:14:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:18 on dist-test-slave-qhn0"
19:14:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.001187 9768 fs_report.cc:383] FS layout report
19:14:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:14:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal
19:14:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal
19:14:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data/data
19:14:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:14:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:14:27.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:14:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:14:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:14:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:14:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:14:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:14:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:14:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:14:27.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:14:27.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.046957 9768 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:14:27.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.052640 9768 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:14:27.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.053052 9768 kserver.cc:163] Server-wide thread pool size limit: 3276
19:14:27.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.144299 9768 rpc_server.cc:307] RPC server started. Bound to: 127.9.47.254:32789
19:14:27.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.144368 9861 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.47.254:32789 every 8 connection(s)
19:14:27.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.147327 9768 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/data/info.pb
19:14:27.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.158334 9407 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu as pid 9768
19:14:27.160 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:14:27.160 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.47.253:38325
19:14:27.161 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
19:14:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.164372 9407 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/logs
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data/info.pb
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:14:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:14:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.47.253:38325
19:14:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.47.253
19:14:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38549
19:14:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.253.keytab
19:14:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.47.253
19:14:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:14:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:14:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:14:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.47.212:35579
19:14:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.47.254:32789,127.9.47.253:38325,127.9.47.252:37663
19:14:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36093/jwks.json
19:14:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:14:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.169538 9862 sys_catalog.cc:263] Verifying existing consensus state
19:14:27.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260620 19:14:27.170459 9862 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:14:27.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.171438 9768 master.cc:537] Master@127.9.47.254:32789 shutting down...
19:14:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.236982 9768 master.cc:552] Master@127.9.47.254:32789 shutdown complete.
19:14:27.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:14:27.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.511113 9864 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:14:27.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.511734 9864 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:14:27.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.512387 9864 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:14:27.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.544443 9864 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:14:27.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.544660 9864 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:14:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.544909 9864 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:14:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.545106 9864 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:14:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.545291 9864 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:14:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.545532 9864 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:14:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.545720 9864 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36093/jwks.json
19:14:27.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.561399 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:27.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.561630 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.583627 9864 master_runner.cc:386] Master server non-default flags:
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.47.212:35579
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.47.254:32789,127.9.47.253:38325,127.9.47.252:37663
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/krb5kdc/kudu_127.9.47.253.keytab
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.47.253
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.47.253:38325
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:14:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36093/jwks.json
19:14:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:14:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:14:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:14:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:14:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:14:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data/info.pb
19:14:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.47.253
19:14:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38549
19:14:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/logs
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 18:30:25 UTC on ccycloud.quasar-sjmjgc.root.comops.site
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457678
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.585115 9864 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:14:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:27.586493 9864 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:14:27.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.606000 9870 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:28.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:28.567473 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:28.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:28.567941 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:28.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.606220 9871 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:28.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:27.608484 9873 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:14:28.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:28.849308 9872 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
19:14:28.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.849387 9864 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:14:28.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.853583 9864 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:14:28.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.856462 9864 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:14:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.857849 9864 hybrid_clock.cc:648] HybridClock initialized: now 1781982868857807 us; error 38 us; skew 500 ppm
19:14:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.858749 9864 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:14:28.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:28 dist-test-slave-qhn0 krb5kdc[9420](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982868, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:14:28.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.864993 9864 init.cc:377] Logged in from keytab as kudu/127.9.47.253@KRBTEST.COM (short username kudu)
19:14:28.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.870323 9864 webserver.cc:460] Webserver started at http://127.9.47.253:38549/ using document root <none> and password file <none>
19:14:28.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.871341 9864 fs_manager.cc:359] Metadata directory not provided
19:14:28.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.871598 9864 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:14:28.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.879544 9864 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
19:14:28.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.883510 9882 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:14:28.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.884431 9864 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
19:14:28.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.884758 9864 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data,/tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal
19:14:28.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4c3072111684769b44985016f14a625"
19:14:28.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 19:14:20 on dist-test-slave-qhn0"
19:14:28.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.887144 9864 fs_report.cc:383] FS layout report
19:14:28.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:14:28.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal
19:14:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal
19:14:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data/data
19:14:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:14:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:14:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:14:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:14:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:14:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:14:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:14:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:14:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:14:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:14:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:14:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.937733 9864 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:14:28.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.943128 9864 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:14:28.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:28.943565 9864 kserver.cc:163] Server-wide thread pool size limit: 3276
19:14:29.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.013036 9864 rpc_server.cc:307] RPC server started. Bound to: 127.9.47.253:38325
19:14:29.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.013163 9933 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.47.253:38325 every 8 connection(s)
19:14:29.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.016045 9864 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/data/info.pb
19:14:29.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.018414 9407 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu as pid 9864
19:14:29.019 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:14:29.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.024168 9936 raft_consensus.cc:482] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:14:29.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.024595 9936 raft_consensus.cc:504] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dcbdae9fa24d48b21275380ffbf243" member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } } peers { permanent_uuid: "e4c3072111684769b44985016f14a625" member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } } peers { permanent_uuid: "a9502b19ca544311a0507617142f5b14" member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } }
19:14:29.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.029920 9934 sys_catalog.cc:263] Verifying existing consensus state
19:14:29.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260620 19:14:29.030939 9934 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:14:29.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.031410 9864 master.cc:537] Master@127.9.47.253:38325 shutting down...
19:14:29.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.034454 9936 leader_election.cc:288] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86dcbdae9fa24d48b21275380ffbf243 (127.9.47.254:32789), e4c3072111684769b44985016f14a625 (127.9.47.253:38325)
19:14:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:29.063473 9715 leader_election.cc:334] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 86dcbdae9fa24d48b21275380ffbf243 (127.9.47.254:32789): Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111)
19:14:29.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:29 dist-test-slave-qhn0 krb5kdc[9420](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982864, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.252@KRBTEST.COM for kudu/127.9.47.253@KRBTEST.COM
19:14:29.086 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.47.252:37663(127.9.47.252:37663)] connecting to peer
19:14:29.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:29.091189 9716 leader_election.cc:334] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4c3072111684769b44985016f14a625 (127.9.47.253:38325): Remote error: Service unavailable: service kudu.consensus.ConsensusService not registered on Master
19:14:29.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.091567 9716 leader_election.cc:302] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a9502b19ca544311a0507617142f5b14; no voters: 86dcbdae9fa24d48b21275380ffbf243, e4c3072111684769b44985016f14a625
19:14:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.093227 9936 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:14:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.152072 9864 master.cc:552] Master@127.9.47.253:38325 shutdown complete.
19:14:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskuIdNT5/test-tmp/mini-kudu-cluster6420230248301424710/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:14:29.172 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:29.174 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:29.192 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.9.47.252:37663(127.9.47.252:37663)] Successfully connected to peer
19:14:29.196 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.202 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.204 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.264 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.265 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.287 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
19:14:29.296 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.9.47.252
19:14:29.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:29 dist-test-slave-qhn0 krb5kdc[9420](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982862, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.47.252@KRBTEST.COM
19:14:29.515 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4dfc0883, L:/127.0.0.1:45272 - R:/127.9.47.252:37663] using SASL/GSSAPI
19:14:29.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:29.547 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:29.549 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:29.552 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.552 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.552 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.555 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.555 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.555 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.556 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:29.566 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:29.567 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:29.568 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.571 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:29.571756 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:29.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:29.572068 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:29.572 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.572 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:29.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.580812 9695 master_runner.cc:186] Error getting master registration for 127.9.47.254:32789: Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111),
19:14:29.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:29.581221 9695 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:14:29.585 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:29.586 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:29.588 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.588 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.590 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.592 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:29.605 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:29.607 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:29.607 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.607 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.608 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.610 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.610 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:29.626 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:29.627 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:29.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:29.645 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:29.646 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:29.648 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.648 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.650 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.651 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.651 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.651 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:29.746 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:29.747 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:29.748 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.748 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.751 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.751 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:29.945 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:29.946 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:29.947 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.948 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:29.953 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.953 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:29.954 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:30.066 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:30.067 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:30.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:30.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:30.070 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:30.070 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:30.071 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:30.071 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:30.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:30.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:30.575724 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:30.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:30.575969 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:30.785 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:30.786 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:30.787 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:30.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:30.788 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:30.790 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:30.790 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:30.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:30.791 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:31.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:31.007078 9392 debug-util.cc:398] Leaking SignalData structure 0x7b08000bc4c0 after lost signal to thread 9267
19:14:31.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:31.007944 9392 debug-util.cc:398] Leaking SignalData structure 0x7b08000af3c0 after lost signal to thread 9395
19:14:31.267 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:31.270 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:31.271 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:31.271 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:31.272 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:31.272 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:31.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:31.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:31.273 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:31.578831 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:31.579135 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:32.195935 9948 raft_consensus.cc:482] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:14:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:32.196372 9948 raft_consensus.cc:504] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dcbdae9fa24d48b21275380ffbf243" member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } } peers { permanent_uuid: "e4c3072111684769b44985016f14a625" member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } } peers { permanent_uuid: "a9502b19ca544311a0507617142f5b14" member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } }
19:14:32.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:32.198217 9948 leader_election.cc:288] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86dcbdae9fa24d48b21275380ffbf243 (127.9.47.254:32789), e4c3072111684769b44985016f14a625 (127.9.47.253:38325)
19:14:32.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:32.200354 9715 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111) [suppressed 5 similar messages]
19:14:32.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:32.203380 9715 leader_election.cc:334] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 86dcbdae9fa24d48b21275380ffbf243 (127.9.47.254:32789): Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111)
19:14:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:32.203830 9716 leader_election.cc:334] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4c3072111684769b44985016f14a625 (127.9.47.253:38325): Network error: Client connection negotiation failed: client connection to 127.9.47.253:38325: connect: Connection refused (error 111)
19:14:32.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:32.204274 9716 leader_election.cc:302] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a9502b19ca544311a0507617142f5b14; no voters: 86dcbdae9fa24d48b21275380ffbf243, e4c3072111684769b44985016f14a625
19:14:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:32.205031 9948 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:14:32.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:32.582125 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:32.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:32.582402 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:32.705 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:32.706 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:32.708 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:32.708 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:32.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:32.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:32.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:32.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:32.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:33.046 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:33.047 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:33.048 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:33.048 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:33.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:33.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:33.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:33.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:33.051 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:33.585659 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:33.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:33.586055 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:34.026 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:34.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:34.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:34.030 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:34.030 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:34.032 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:34.032 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:34.032 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:34.032 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:34.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:34 dist-test-slave-qhn0 krb5kdc[9420](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982874, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:14:34.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:34.242220 9710 init.cc:295] Successfully reacquired a new kerberos TGT
19:14:34.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:34.584208 9695 master_runner.cc:186] Error getting master registration for 127.9.47.254:32789: Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111),
19:14:34.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:34.584573 9695 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:14:34.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:34.588735 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:34.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:34.588943 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:35.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:35.591760 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:35.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:35.592017 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:36.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:36.353608 9959 raft_consensus.cc:482] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:14:36.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:36.354225 9959 raft_consensus.cc:504] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dcbdae9fa24d48b21275380ffbf243" member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } } peers { permanent_uuid: "e4c3072111684769b44985016f14a625" member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } } peers { permanent_uuid: "a9502b19ca544311a0507617142f5b14" member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } }
19:14:36.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:36.356199 9959 leader_election.cc:288] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86dcbdae9fa24d48b21275380ffbf243 (127.9.47.254:32789), e4c3072111684769b44985016f14a625 (127.9.47.253:38325)
19:14:36.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:36.361231 9715 leader_election.cc:334] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 86dcbdae9fa24d48b21275380ffbf243 (127.9.47.254:32789): Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111)
19:14:36.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:36.361805 9716 leader_election.cc:334] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4c3072111684769b44985016f14a625 (127.9.47.253:38325): Network error: Client connection negotiation failed: client connection to 127.9.47.253:38325: connect: Connection refused (error 111)
19:14:36.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:36.362164 9716 leader_election.cc:302] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a9502b19ca544311a0507617142f5b14; no voters: 86dcbdae9fa24d48b21275380ffbf243, e4c3072111684769b44985016f14a625
19:14:36.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:36.362898 9959 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:14:36.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:36.594688 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:36.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:36.594930 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:37.065 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:37.067 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:37.068 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:37.068 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:37.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:37.070 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:37.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:37.071 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:37.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:37.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:37.597597 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:37.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:37.597958 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:37.646 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:37.646 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:37.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:37.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:37.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:37.651 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:37.651 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:37.651 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:37.651 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:38.600889 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:38.601150 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:39.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:39.586934 9695 master_runner.cc:186] Error getting master registration for 127.9.47.254:32789: Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111),
19:14:39.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:39.587303 9695 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:14:39.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:39.603983 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:39.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:39.604274 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:40.046 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:40.046 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:40.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:40.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:40.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:40.050 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:40.050 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:40.050 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:40.050 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:40.445 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:40.447 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:40.447 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:40.448 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:40.448 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:40.450 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:40.450 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:40.450 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:40.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:40.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:40.607342 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:40.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:40.607589 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:41.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:41.610407 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:41.610704 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:41.785 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:41.787 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:41.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:41.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:41.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:41.790 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:41.790 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:41.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:41.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:41.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:41.885563 8755 debug-util.cc:398] Leaking SignalData structure 0x7b0800082a00 after lost signal to thread 8692
19:14:41.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:41.886129 8755 debug-util.cc:398] Leaking SignalData structure 0x7b0800052080 after lost signal to thread 8758
19:14:42.466 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:42.466 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:42.468 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:42.468 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:42.468 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:42.470 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:42.470 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:42.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:42.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:42.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:42.613483 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:42.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:42.613758 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:42.626 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:42.626 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:42.627 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:42.627 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:42.627 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:42.628 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:42.628 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:42.628 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:42.629 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:43.066 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:43.066 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:43.067 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:43.067 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:43.067 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:43.067 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:43.067 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:43.068 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:43.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:43.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:43.616848 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:43.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:43.617762 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:44.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:44 dist-test-slave-qhn0 krb5kdc[9420](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982884, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:14:44.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:44.244601 9710 init.cc:295] Successfully reacquired a new kerberos TGT
19:14:44.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:44.590283 9695 master_runner.cc:186] Error getting master registration for 127.9.47.254:32789: Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111),
19:14:44.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:44.590658 9695 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:14:44.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:44.620553 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:44.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:44.620770 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:44.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:44.788717 9972 raft_consensus.cc:482] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:14:44.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:44.789132 9972 raft_consensus.cc:504] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86dcbdae9fa24d48b21275380ffbf243" member_type: VOTER last_known_addr { host: "127.9.47.254" port: 32789 } } peers { permanent_uuid: "e4c3072111684769b44985016f14a625" member_type: VOTER last_known_addr { host: "127.9.47.253" port: 38325 } } peers { permanent_uuid: "a9502b19ca544311a0507617142f5b14" member_type: VOTER last_known_addr { host: "127.9.47.252" port: 37663 } }
19:14:44.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:44.790709 9972 leader_election.cc:288] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86dcbdae9fa24d48b21275380ffbf243 (127.9.47.254:32789), e4c3072111684769b44985016f14a625 (127.9.47.253:38325)
19:14:44.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:44.792343 9715 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111) [suppressed 3 similar messages]
19:14:44.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:44.795181 9715 leader_election.cc:334] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 86dcbdae9fa24d48b21275380ffbf243 (127.9.47.254:32789): Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111)
19:14:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:44.795780 9716 leader_election.cc:334] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e4c3072111684769b44985016f14a625 (127.9.47.253:38325): Network error: Client connection negotiation failed: client connection to 127.9.47.253:38325: connect: Connection refused (error 111)
19:14:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:44.796088 9716 leader_election.cc:302] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a9502b19ca544311a0507617142f5b14; no voters: 86dcbdae9fa24d48b21275380ffbf243, e4c3072111684769b44985016f14a625
19:14:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:44.796733 9972 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:14:45.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:45.623456 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:45.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:45.623689 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:46.085 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:46.086 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:46.087 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:46.087 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:46.087 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:46.087 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:46.087 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:46.087 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:46.089 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:46.326 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:46.327 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:46.327 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:46.327 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:46.327 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:46.328 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:46.328 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:46.328 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:46.329 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:46.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:46.626369 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:46.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:46.626614 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:47.629503 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:47.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:47.629825 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:47.706 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:47.707 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:47.707 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:47.707 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:47.707 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:47.708 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:47.708 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:47.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:47.712 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:48.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:48.038408 9252 debug-util.cc:398] Leaking SignalData structure 0x7b08000be480 after lost signal to thread 9124
19:14:48.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:48.039222 9252 debug-util.cc:398] Leaking SignalData structure 0x7b08000afb80 after lost signal to thread 9255
19:14:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:48.634581 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:48.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:48.634946 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:49.166 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:49.166 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:49.167 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:49.167 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:49.167 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:49.168 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:49.168 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:49.168 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:49.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:49.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:49.593400 9695 master_runner.cc:186] Error getting master registration for 127.9.47.254:32789: Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111),
19:14:49.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:49.593775 9695 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:14:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:49.639497 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:49.639837 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:50.306 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:50.307 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:50.307 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:50.307 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:50.308 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:50.308 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:50.308 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:50.308 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:50.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:50.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:50.642705 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:50.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:50.642995 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:51.525 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:51.526 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:51.526 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:51.527 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:51.527 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:51.527 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:51.527 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:51.527 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:51.529 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:51.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:51.645982 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:51.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:51.646306 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:52.486 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:52.487 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:52.487 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:52.487 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:52.488 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:52.488 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:52.488 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:52.488 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:52.489 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:52.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:52.650220 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:52.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:52.650547 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:53.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:53.655362 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:53.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:53.655756 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:54.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 19:14:54 dist-test-slave-qhn0 krb5kdc[9420](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781982894, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.47.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:14:54.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:54.246887 9710 init.cc:295] Successfully reacquired a new kerberos TGT
19:14:54.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:54.597040 9695 master_runner.cc:186] Error getting master registration for 127.9.47.254:32789: Network error: Client connection negotiation failed: client connection to 127.9.47.254:32789: connect: Connection refused (error 111),
19:14:54.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:54.597471 9695 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:14:54.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:54.658464 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:54.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:54.658722 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:54.925 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:54.927 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:54.927 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:54.928 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:54.928 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:54.930 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:54.930 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:54.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:54.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:55.661027 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:55.661274 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:56.663857 9781 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a9502b19ca544311a0507617142f5b14: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:14:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 19:14:56.664084 9781 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:14:56.986 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:56.987 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:56.987 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:56.987 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:56.987 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:56.987 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:56.988 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:56.988 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:56.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:57.146 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.254:32789(127.9.47.254:32789)] connecting to peer
19:14:57.147 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.47.253:38325(127.9.47.253:38325)] connecting to peer
19:14:57.147 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:57.148 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.47.254:32789(127.9.47.254:32789)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:57.148 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:57.148 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.253:38325(127.9.47.253:38325)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:57.148 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.254:32789: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789
19:14:57.148 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.47.253:38325: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
19:14:57.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325; will retry
19:14:57.155 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=33, TimeoutTracker(timeout=30000, elapsed=28127), Traces: [0ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [174ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [236ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [497ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [523ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [523ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [526ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [527ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [536ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [540ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [541ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [543ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [559ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [561ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [564ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [578ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [580ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [581ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [582ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [600ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [601ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [602ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [603ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [619ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [621ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [622ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [623ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [715ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [720ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [721ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [722ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [723ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [919ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [922ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [924ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [925ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1036ms] refreshing cache from master, [1036ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [1036ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1043ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1756ms] refreshing cache from master, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [1757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [1759ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1762ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [2236ms] refreshing cache from master, [2236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [2238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [2241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [2242ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [2243ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [2244ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [2244ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [3676ms] refreshing cache from master, [3676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [3676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [3677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [3679ms] too many traces: truncated at 100 traces, deferred=Deferred@2083409068(state=PENDING, result=null, callback=(continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183) -> (continuation of Deferred@890691364 after retry RPC after error@1821322821) -> (continuation of Deferred@2015043681 after retry RPC after error@834812713) -> (continuation of Deferred@841887178 after retry RPC after error@658952396) -> (continuation of Deferred@117437549 after retry RPC after error@650419432) -> (continuation of Deferred@1707582855 after retry RPC after error@1956043075) -> (continuation of Deferred@221447968 after retry RPC after error@252901810) -> (continuation of Deferred@1470205514 after retry RPC after error@364845372) -> (continuation of Deferred@2102956 after retry RPC after error@409448563) -> (continuation of Deferred@1139785451 after retry RPC after error@779368787) -> (continuation of Deferred@778161991 after retry RPC after error@195036880) -> (continuation of Deferred@898038185 after retry RPC after error@976740587), errback=(continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183) -> (continuation of Deferred@890691364 after retry RPC after error@1821322821) -> (continuation of Deferred@2015043681 after retry RPC after error@834812713) -> (continuation of Deferred@841887178 after retry RPC after error@658952396) -> (continuation of Deferred@117437549 after retry RPC after error@650419432) -> (continuation of Deferred@1707582855 after retry RPC after error@1956043075) -> (continuation of Deferred@221447968 after retry RPC after error@252901810) -> (continuation of Deferred@1470205514 after retry RPC after error@364845372) -> (continuation of Deferred@2102956 after retry RPC after error@409448563) -> (continuation of Deferred@1139785451 after retry RPC after error@779368787) -> (continuation of Deferred@778161991 after retry RPC after error@195036880) -> (continuation of Deferred@898038185 after retry RPC after error@976740587)))
19:14:57.160 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.47.252:37663(127.9.47.252:37663)] disconnecting while in state READY
19:14:57.161 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.47.254:32789(127.9.47.254:32789)] disconnecting while in state TERMINATED
19:14:57.161 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.47.253:38325(127.9.47.253:38325)] disconnecting while in state TERMINATED
19:14:57.166 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:14:57.169 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:14:57.173 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.9.47.252:37663(127.9.47.252:37663)] handling channelInactive
19:14:57.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.47.252:37663(127.9.47.252:37663)] cleaning up while in state READY due to: connection closed
19:14:57.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:57.177744 9407 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu with pid 9695
19:14:58.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T19:14:58Z chronyd exiting
19:14:59.384 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:14:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:59.386190 8679 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu with pid 8971
19:14:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:59.410916 8679 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu with pid 9118
19:14:59.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:59.437995 8679 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu with pid 9266
19:14:59.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:59.469154 8679 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu with pid 8691
19:14:59.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:59.499769 8679 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu with pid 8761
19:14:59.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 19:14:59.528138 8679 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskuIdNT5/build/tsan/bin/kudu with pid 8836
19:15:00.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T19:15:00Z chronyd exiting
19:15:01.699 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=33, TimeoutTracker(timeout=30000, elapsed=28127), Traces: [0ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [174ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [236ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [497ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [523ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [523ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [526ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [527ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [536ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [540ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [541ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [543ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [559ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [561ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [564ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [578ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [580ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [581ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [582ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [600ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [601ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [602ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [603ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [619ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [621ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [622ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [623ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [715ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [720ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [721ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [722ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [723ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [919ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [922ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [924ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [925ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1036ms] refreshing cache from master, [1036ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [1036ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1043ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1756ms] refreshing cache from master, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [1757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [1759ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1762ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [2236ms] refreshing cache from master, [2236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [2238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [2241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [2242ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [2243ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [2244ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [2244ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [3676ms] refreshing cache from master, [3676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [3676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [3677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [3679ms] too many traces: truncated at 100 traces, deferred=Deferred@2083409068(state=PENDING, result=null, callback=(continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183) -> (continuation of Deferred@890691364 after retry RPC after error@1821322821) -> (continuation of Deferred@2015043681 after retry RPC after error@834812713) -> (continuation of Deferred@841887178 after retry RPC after error@658952396) -> (continuation of Deferred@117437549 after retry RPC after error@650419432) -> (continuation of Deferred@1707582855 after retry RPC after error@1956043075) -> (continuation of Deferred@221447968 after retry RPC after error@252901810) -> (continuation of Deferred@1470205514 after retry RPC after error@364845372) -> (continuation of Deferred@2102956 after retry RPC after error@409448563) -> (continuation of Deferred@1139785451 after retry RPC after error@779368787) -> (continuation of Deferred@778161991 after retry RPC after error@195036880) -> (continuation of Deferred@898038185 after retry RPC after error@976740587), errback=(continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183) -> (continuation of Deferred@890691364 after retry RPC after error@1821322821) -> (continuation of Deferred@2015043681 after retry RPC after error@834812713) -> (continuation of Deferred@841887178 after retry RPC after error@658952396) -> (continuation of Deferred@117437549 after retry RPC after error@650419432) -> (continuation of Deferred@1707582855 after retry RPC after error@1956043075) -> (continuation of Deferred@221447968 after retry RPC after error@252901810) -> (continuation of Deferred@1470205514 after retry RPC after error@364845372) -> (continuation of Deferred@2102956 after retry RPC after error@409448563) -> (continuation of Deferred@1139785451 after retry RPC after error@779368787) -> (continuation of Deferred@778161991 after retry RPC after error@195036880) -> (continuation of Deferred@898038185 after retry RPC after error@976740587)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.47.253:38325
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=33, TimeoutTracker(timeout=30000, elapsed=28127), Traces: [0ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [174ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [236ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [497ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [523ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [523ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [526ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [527ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [536ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [540ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [541ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [543ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [559ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [561ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [564ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [578ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [580ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [581ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [582ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [600ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [601ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [602ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [603ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [619ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [621ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [622ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [623ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [715ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [720ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [721ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [722ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [723ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [919ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [922ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [924ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [925ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1036ms] refreshing cache from master, [1036ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [1036ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1043ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1756ms] refreshing cache from master, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [1757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [1759ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1762ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [2236ms] refreshing cache from master, [2236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [2238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [2241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [2242ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [2243ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [2244ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [2244ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [3676ms] refreshing cache from master, [3676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [3676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [3677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [3679ms] too many traces: truncated at 100 traces, deferred=Deferred@2083409068(state=PENDING, result=null, callback=(continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183) -> (continuation of Deferred@890691364 after retry RPC after error@1821322821) -> (continuation of Deferred@2015043681 after retry RPC after error@834812713) -> (continuation of Deferred@841887178 after retry RPC after error@658952396) -> (continuation of Deferred@117437549 after retry RPC after error@650419432) -> (continuation of Deferred@1707582855 after retry RPC after error@1956043075) -> (continuation of Deferred@221447968 after retry RPC after error@252901810) -> (continuation of Deferred@1470205514 after retry RPC after error@364845372) -> (continuation of Deferred@2102956 after retry RPC after error@409448563) -> (continuation of Deferred@1139785451 after retry RPC after error@779368787) -> (continuation of Deferred@778161991 after retry RPC after error@195036880) -> (continuation of Deferred@898038185 after retry RPC after error@976740587), errback=(continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183) -> (continuation of Deferred@890691364 after retry RPC after error@1821322821) -> (continuation of Deferred@2015043681 after retry RPC after error@834812713) -> (continuation of Deferred@841887178 after retry RPC after error@658952396) -> (continuation of Deferred@117437549 after retry RPC after error@650419432) -> (continuation of Deferred@1707582855 after retry RPC after error@1956043075) -> (continuation of Deferred@221447968 after retry RPC after error@252901810) -> (continuation of Deferred@1470205514 after retry RPC after error@364845372) -> (continuation of Deferred@2102956 after retry RPC after error@409448563) -> (continuation of Deferred@1139785451 after retry RPC after error@779368787) -> (continuation of Deferred@778161991 after retry RPC after error@195036880) -> (continuation of Deferred@898038185 after retry RPC after error@976740587)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.47.253:38325
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
19:15:02.089 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
19:15:02.089 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=33, TimeoutTracker(timeout=30000, elapsed=28127), Traces: [0ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [174ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [236ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [497ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [523ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [523ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [526ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [527ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [536ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [540ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [541ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [543ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [559ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [561ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [564ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [578ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [580ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [581ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [582ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [600ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [601ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [602ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [603ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [619ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [621ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [622ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [623ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [715ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [720ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [721ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [722ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [723ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [919ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [922ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [924ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [925ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1036ms] refreshing cache from master, [1036ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [1036ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1043ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1756ms] refreshing cache from master, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [1757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [1759ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1762ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [2236ms] refreshing cache from master, [2236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [2238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [2241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [2242ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [2243ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [2244ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [2244ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [3676ms] refreshing cache from master, [3676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [3676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [3677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [3679ms] too many traces: truncated at 100 traces, deferred=Deferred@2083409068(state=PENDING, result=null, callback=(continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183) -> (continuation of Deferred@890691364 after retry RPC after error@1821322821) -> (continuation of Deferred@2015043681 after retry RPC after error@834812713) -> (continuation of Deferred@841887178 after retry RPC after error@658952396) -> (continuation of Deferred@117437549 after retry RPC after error@650419432) -> (continuation of Deferred@1707582855 after retry RPC after error@1956043075) -> (continuation of Deferred@221447968 after retry RPC after error@252901810) -> (continuation of Deferred@1470205514 after retry RPC after error@364845372) -> (continuation of Deferred@2102956 after retry RPC after error@409448563) -> (continuation of Deferred@1139785451 after retry RPC after error@779368787) -> (continuation of Deferred@778161991 after retry RPC after error@195036880) -> (continuation of Deferred@898038185 after retry RPC after error@976740587), errback=(continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183) -> (continuation of Deferred@890691364 after retry RPC after error@1821322821) -> (continuation of Deferred@2015043681 after retry RPC after error@834812713) -> (continuation of Deferred@841887178 after retry RPC after error@658952396) -> (continuation of Deferred@117437549 after retry RPC after error@650419432) -> (continuation of Deferred@1707582855 after retry RPC after error@1956043075) -> (continuation of Deferred@221447968 after retry RPC after error@252901810) -> (continuation of Deferred@1470205514 after retry RPC after error@364845372) -> (continuation of Deferred@2102956 after retry RPC after error@409448563) -> (continuation of Deferred@1139785451 after retry RPC after error@779368787) -> (continuation of Deferred@778161991 after retry RPC after error@195036880) -> (continuation of Deferred@898038185 after retry RPC after error@976740587)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.47.253:38325
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=33, TimeoutTracker(timeout=30000, elapsed=28127), Traces: [0ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [174ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [236ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [497ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [523ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [523ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [526ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [527ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [536ms] refreshing cache from master, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [540ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [541ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [543ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [556ms] refreshing cache from master, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [559ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [561ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [562ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [564ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [578ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [580ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [581ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [582ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [600ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [601ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [602ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [603ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [615ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [619ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [621ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [622ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [623ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [715ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [720ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [721ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [722ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [723ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [919ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [922ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [924ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [925ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1036ms] refreshing cache from master, [1036ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [1036ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [1041ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1043ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1756ms] refreshing cache from master, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [1756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [1757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [1759ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [1761ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [1762ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [2236ms] refreshing cache from master, [2236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [2238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [2241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [2242ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.252:37663: OK, [2243ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.254:32789: Network error: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789, [2244ms] Sub RPC ConnectToMaster: received response from server master-127.9.47.253:38325: Network error: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [2244ms] delaying RPC due to: Service unavailable: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325, [3676ms] refreshing cache from master, [3676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.252:37663, [3676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.254:32789, [3677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.47.253:38325, [3679ms] too many traces: truncated at 100 traces, deferred=Deferred@2083409068(state=PENDING, result=null, callback=(continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183) -> (continuation of Deferred@890691364 after retry RPC after error@1821322821) -> (continuation of Deferred@2015043681 after retry RPC after error@834812713) -> (continuation of Deferred@841887178 after retry RPC after error@658952396) -> (continuation of Deferred@117437549 after retry RPC after error@650419432) -> (continuation of Deferred@1707582855 after retry RPC after error@1956043075) -> (continuation of Deferred@221447968 after retry RPC after error@252901810) -> (continuation of Deferred@1470205514 after retry RPC after error@364845372) -> (continuation of Deferred@2102956 after retry RPC after error@409448563) -> (continuation of Deferred@1139785451 after retry RPC after error@779368787) -> (continuation of Deferred@778161991 after retry RPC after error@195036880) -> (continuation of Deferred@898038185 after retry RPC after error@976740587), errback=(continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858) -> (continuation of Deferred@1853473501 after retry RPC after error@2020206422) -> (continuation of Deferred@115260359 after retry RPC after error@241279275) -> (continuation of Deferred@774825232 after retry RPC after error@424338644) -> (continuation of Deferred@1790534841 after retry RPC after error@1775429692) -> (continuation of Deferred@814585459 after retry RPC after error@2125441266) -> (continuation of Deferred@1595049778 after retry RPC after error@570361828) -> (continuation of Deferred@2099061670 after retry RPC after error@2121727183) -> (continuation of Deferred@890691364 after retry RPC after error@1821322821) -> (continuation of Deferred@2015043681 after retry RPC after error@834812713) -> (continuation of Deferred@841887178 after retry RPC after error@658952396) -> (continuation of Deferred@117437549 after retry RPC after error@650419432) -> (continuation of Deferred@1707582855 after retry RPC after error@1956043075) -> (continuation of Deferred@221447968 after retry RPC after error@252901810) -> (continuation of Deferred@1470205514 after retry RPC after error@364845372) -> (continuation of Deferred@2102956 after retry RPC after error@409448563) -> (continuation of Deferred@1139785451 after retry RPC after error@779368787) -> (continuation of Deferred@778161991 after retry RPC after error@195036880) -> (continuation of Deferred@898038185 after retry RPC after error@976740587)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.47.252:37663,127.9.47.254:32789,127.9.47.253:38325) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.254:32789(127.9.47.254:32789): Connection refused: /127.9.47.254:32789,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.47.253:38325(127.9.47.253:38325): Connection refused: /127.9.47.253:38325
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.47.253:38325
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)