06:43:13.454 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
06:43:13.465 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskUfq9I_/test-tmp
06:43:13.468 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
06:43:13.471 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskUfq9I_/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
06:43:13.699 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
06:43:13.861 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:43:13.862 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
06:43:13.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:13.866057 6630 mini_oidc.cc:152] Starting JWKS server
06:43:13.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:13.870252 6630 webserver.cc:460] Webserver started at http://127.0.0.1:35495/ using document root <none> and password file <none>
06:43:13.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:13.870725 6630 mini_oidc.cc:189] Starting OIDC Discovery server
06:43:13.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:13.872548 6630 webserver.cc:460] Webserver started at http://127.0.0.1:36393/ using document root <none> and password file <none>
06:43:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T06:43:13Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:43:13.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T06:43:13Z Disabled control of system clock
06:43:13.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:13.921083 6630 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:13.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:13.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/wal
06:43:13.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/data
06:43:13.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:43:13.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:43:13.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:43:13.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:13.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:13.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:13.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/logs
06:43:13.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/data/info.pb
06:43:13.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:13.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:43:13.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:43:13.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:43:13.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:43:13.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:13.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:13.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:13.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:43:13.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:43:13.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:13.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:13.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.121.190:34235
06:43:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.121.190
06:43:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.121.148:45515
06:43:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.121.190:34235,127.6.121.189:46423,127.6.121.188:44129
06:43:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:13.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35495/jwks.json with env {}
06:43:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:14.220140 6642 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:43:14.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:14.220703 6642 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:43:14.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:14.221328 6642 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:43:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:14.260725 6642 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:43:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:14.261015 6642 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:43:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:14.261215 6642 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:43:14.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:14.261404 6642 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:43:14.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:14.261730 6642 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:43:14.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:14.261891 6642 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35495/jwks.json
06:43:14.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:14.296200 6642 master_runner.cc:386] Master server non-default flags:
06:43:14.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:14.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.121.148:45515
06:43:14.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:14.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:14.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/data
06:43:14.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/wal
06:43:14.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:14.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.121.190:34235,127.6.121.189:46423,127.6.121.188:44129
06:43:14.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:14.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:14.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:14.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.121.190:34235
06:43:14.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:14.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:43:14.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:14.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35495/jwks.json
06:43:14.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:14.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:14.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/data/info.pb
06:43:14.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.121.190
06:43:14.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:14.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:43:14.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:14.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:43:14.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:43:14.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:14.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/logs
06:43:14.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:14.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:43:14.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:43:14.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:43:14.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:43:14.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:43:14.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:43:14.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:43:14.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:43:14.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:43:14.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:14.297459 6642 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:43:14.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:14.299373 6642 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:43:14.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:14.317343 6648 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:15.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:14.317493 6651 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:15.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:14.317710 6649 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:15.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:14.318513 6642 server_base.cc:1034] running on GCE node
06:43:15.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.550402 6642 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:43:15.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.553606 6642 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:43:15.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.554998 6642 hybrid_clock.cc:648] HybridClock initialized: now 1790664195554956 us; error 61 us; skew 500 ppm
06:43:15.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.555756 6642 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:43:15.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.566354 6642 webserver.cc:460] Webserver started at http://127.6.121.190:36423/ using document root <none> and password file <none>
06:43:15.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.567281 6642 fs_manager.cc:359] Metadata directory not provided
06:43:15.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.567466 6642 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:43:15.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.567831 6642 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:43:15.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.572755 6642 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/data/instance:
06:43:15.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27b6f55e7c514e47990ba697c9996299"
06:43:15.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:15 on dist-test-slave-vcpq"
06:43:15.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.573905 6642 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/wal/instance:
06:43:15.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27b6f55e7c514e47990ba697c9996299"
06:43:15.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:15 on dist-test-slave-vcpq"
06:43:15.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.581244 6642 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
06:43:15.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.586658 6658 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:43:15.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.587860 6642 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:43:15.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.588143 6642 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/data,/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/wal
06:43:15.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27b6f55e7c514e47990ba697c9996299"
06:43:15.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:15 on dist-test-slave-vcpq"
06:43:15.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.588439 6642 fs_report.cc:383] FS layout report
06:43:15.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:43:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/wal
06:43:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/wal
06:43:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/data/data
06:43:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:43:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:43:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:43:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:43:15.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:43:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:43:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:43:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:43:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:43:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:43:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:43:15.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.633769 6642 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:43:15.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.635215 6642 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:43:15.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.635607 6642 kserver.cc:163] Server-wide thread pool size limit: 3276
06:43:15.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.704488 6642 rpc_server.cc:307] RPC server started. Bound to: 127.6.121.190:34235
06:43:15.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.704664 6709 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.121.190:34235 every 8 connection(s)
06:43:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.707230 6642 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/data/info.pb
06:43:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.711961 6630 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu as pid 6642
06:43:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.712431 6630 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-0/wal/instance
06:43:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.713346 6710 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:43:15.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.715982 6630 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:15.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:15.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/wal
06:43:15.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/data
06:43:15.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:43:15.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:43:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:43:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/logs
06:43:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/data/info.pb
06:43:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:43:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:43:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:43:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:43:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:43:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:43:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.121.189:46423
06:43:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.121.189
06:43:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.121.148:45515
06:43:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.121.190:34235,127.6.121.189:46423,127.6.121.188:44129
06:43:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35495/jwks.json with env {}
06:43:15.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.726997 6710 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } has no permanent_uuid. Determining permanent_uuid...
06:43:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.757683 6710 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } has no permanent_uuid. Determining permanent_uuid...
06:43:15.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:15.759984 6659 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.121.189:46423: connect: Connection refused (error 111)
06:43:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:15.763978 6710 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.189:46423: Network error: Client connection negotiation failed: client connection to 127.6.121.189:46423: connect: Connection refused (error 111)
06:43:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.813393 6710 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } attempt: 1
06:43:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:15.817354 6710 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.189:46423: Network error: Client connection negotiation failed: client connection to 127.6.121.189:46423: connect: Connection refused (error 111)
06:43:15.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.885756 6710 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } attempt: 2
06:43:15.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:15.889583 6710 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.189:46423: Network error: Client connection negotiation failed: client connection to 127.6.121.189:46423: connect: Connection refused (error 111)
06:43:15.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:15.980994 6710 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } attempt: 3
06:43:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:15.984393 6710 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.189:46423: Network error: Client connection negotiation failed: client connection to 127.6.121.189:46423: connect: Connection refused (error 111)
06:43:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.025573 6712 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:43:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.026110 6712 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:43:16.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.026701 6712 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:43:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.057091 6712 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:43:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.057377 6712 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:43:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.057672 6712 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:43:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.057982 6712 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:43:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.058317 6712 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:43:16.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.058562 6712 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35495/jwks.json
06:43:16.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:16.093307 6712 master_runner.cc:386] Master server non-default flags:
06:43:16.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.121.148:45515
06:43:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/data
06:43:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/wal
06:43:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.121.190:34235,127.6.121.189:46423,127.6.121.188:44129
06:43:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.121.189:46423
06:43:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35495/jwks.json
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/data/info.pb
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.121.189
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/logs
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:43:16.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:43:16.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:43:16.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:43:16.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:43:16.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:43:16.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:43:16.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:43:16.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:16.094630 6712 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:43:16.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:16.096735 6712 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:43:16.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.114099 6722 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:16.127815 6710 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } attempt: 4
06:43:16.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.131608 6710 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.189:46423: Network error: Client connection negotiation failed: client connection to 127.6.121.189:46423: connect: Connection refused (error 111)
06:43:16.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:16.431114 6710 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } attempt: 5
06:43:16.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.435978 6710 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.189:46423: Network error: Client connection negotiation failed: client connection to 127.6.121.189:46423: connect: Connection refused (error 111)
06:43:16.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:16.983366 6710 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } attempt: 6
06:43:16.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.989895 6710 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.189:46423: Network error: Client connection negotiation failed: client connection to 127.6.121.189:46423: connect: Connection refused (error 111)
06:43:17.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.114156 6721 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:17.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:16.115654 6724 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:17.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:16.117774 6712 server_base.cc:1034] running on GCE node
06:43:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.304175 6712 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:43:17.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.306697 6712 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:43:17.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.308106 6712 hybrid_clock.cc:648] HybridClock initialized: now 1790664197308031 us; error 99 us; skew 500 ppm
06:43:17.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.308948 6712 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:43:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.315646 6712 webserver.cc:460] Webserver started at http://127.6.121.189:35473/ using document root <none> and password file <none>
06:43:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.316576 6712 fs_manager.cc:359] Metadata directory not provided
06:43:17.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.316751 6712 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:43:17.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.317131 6712 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:43:17.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.321879 6712 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/data/instance:
06:43:17.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a5e73069e934b468be51251ac8e357c"
06:43:17.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:17 on dist-test-slave-vcpq"
06:43:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.323112 6712 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/wal/instance:
06:43:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a5e73069e934b468be51251ac8e357c"
06:43:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:17 on dist-test-slave-vcpq"
06:43:17.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.330138 6712 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
06:43:17.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.335719 6733 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:43:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.336900 6712 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
06:43:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.337234 6712 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/data,/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/wal
06:43:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a5e73069e934b468be51251ac8e357c"
06:43:17.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:17 on dist-test-slave-vcpq"
06:43:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.337568 6712 fs_report.cc:383] FS layout report
06:43:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:43:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/wal
06:43:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/wal
06:43:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/data/data
06:43:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:43:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:43:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:43:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:43:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:43:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:43:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:43:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:43:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:43:17.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:43:17.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:43:17.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.381013 6712 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:43:17.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.382522 6712 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:43:17.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.383006 6712 kserver.cc:163] Server-wide thread pool size limit: 3276
06:43:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.457906 6712 rpc_server.cc:307] RPC server started. Bound to: 127.6.121.189:46423
06:43:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.457988 6784 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.121.189:46423 every 8 connection(s)
06:43:17.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.461022 6712 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/data/info.pb
06:43:17.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.465211 6630 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu as pid 6712
06:43:17.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.465654 6630 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-1/wal/instance
06:43:17.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.466995 6785 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.468640 6630 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/wal
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/data
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/logs
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/data/info.pb
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:43:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.121.188:44129
06:43:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.121.188
06:43:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.121.148:45515
06:43:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.121.190:34235,127.6.121.189:46423,127.6.121.188:44129
06:43:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35495/jwks.json with env {}
06:43:17.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.480387 6785 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } has no permanent_uuid. Determining permanent_uuid...
06:43:17.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.502666 6785 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } has no permanent_uuid. Determining permanent_uuid...
06:43:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.518131 6785 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } has no permanent_uuid. Determining permanent_uuid...
06:43:17.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.520381 6737 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:17.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.525204 6785 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.188:44129: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:17.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.574687 6785 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } attempt: 1
06:43:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.578312 6785 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.188:44129: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.646811 6785 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } attempt: 2
06:43:17.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.650671 6785 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.188:44129: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:17.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.742130 6785 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } attempt: 3
06:43:17.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.745694 6785 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.188:44129: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:17.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.783129 6787 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:43:17.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.783756 6787 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:43:17.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.784359 6787 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:43:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.815114 6787 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:43:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.815392 6787 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:43:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.815587 6787 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:43:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.815765 6787 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:43:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.816027 6787 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:43:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.816175 6787 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35495/jwks.json
06:43:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.850965 6787 master_runner.cc:386] Master server non-default flags:
06:43:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.121.148:45515
06:43:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/data
06:43:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/wal
06:43:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.121.190:34235,127.6.121.189:46423,127.6.121.188:44129
06:43:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.121.188:44129
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35495/jwks.json
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/data/info.pb
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.121.188
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/logs
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:43:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:43:17.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:43:17.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:43:17.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:43:17.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:43:17.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:43:17.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:43:17.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:43:17.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.852187 6787 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:43:17.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.854250 6787 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:43:17.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.874315 6798 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:17.889063 6785 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } attempt: 4
06:43:17.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.893515 6785 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.188:44129: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:18.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:18.050541 6710 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } attempt: 7
06:43:18.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:18.068250 6710 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } has no permanent_uuid. Determining permanent_uuid...
06:43:18.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:18.074579 6710 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.188:44129: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:18.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:18.133047 6710 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } attempt: 1
06:43:18.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:18.137164 6710 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.188:44129: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:18.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:18.192883 6785 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } attempt: 5
06:43:18.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:18.197342 6785 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.188:44129: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:18.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:18.218588 6710 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } attempt: 2
06:43:18.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:18.223031 6710 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.188:44129: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:18.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:18.308414 6710 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } attempt: 3
06:43:18.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:18.312274 6710 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.188:44129: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:18.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:18.452924 6710 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } attempt: 4
06:43:18.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:18.459146 6710 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.188:44129: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:18.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:18.742790 6710 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } attempt: 5
06:43:18.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:18.744861 6785 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } attempt: 6
06:43:18.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:18.746624 6710 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.188:44129: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:18.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:18.752671 6785 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.121.188:44129: Network error: Client connection negotiation failed: client connection to 127.6.121.188:44129: connect: Connection refused (error 111)
06:43:19.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.874349 6797 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:19.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:17.875463 6800 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:19.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:19.095717 6799 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:43:19.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.095842 6787 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:43:19.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.100160 6787 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:43:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.103102 6787 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:43:19.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.104504 6787 hybrid_clock.cc:648] HybridClock initialized: now 1790664199104476 us; error 41 us; skew 500 ppm
06:43:19.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.105425 6787 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:43:19.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.112963 6787 webserver.cc:460] Webserver started at http://127.6.121.188:46461/ using document root <none> and password file <none>
06:43:19.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.113916 6787 fs_manager.cc:359] Metadata directory not provided
06:43:19.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.114138 6787 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:43:19.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.114683 6787 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:43:19.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.119815 6787 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/data/instance:
06:43:19.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9787b799fe9e4859842e8e57eab53257"
06:43:19.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:19 on dist-test-slave-vcpq"
06:43:19.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.121088 6787 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/wal/instance:
06:43:19.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9787b799fe9e4859842e8e57eab53257"
06:43:19.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:19 on dist-test-slave-vcpq"
06:43:19.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.128542 6787 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
06:43:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.133836 6812 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:43:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.134934 6787 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
06:43:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.135349 6787 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/data,/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/wal
06:43:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9787b799fe9e4859842e8e57eab53257"
06:43:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:19 on dist-test-slave-vcpq"
06:43:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.135699 6787 fs_report.cc:383] FS layout report
06:43:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:43:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/wal
06:43:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/wal
06:43:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/data/data
06:43:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:43:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:43:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:43:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:43:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:43:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:43:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:43:19.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:43:19.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:43:19.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:43:19.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:43:19.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.166894 6787 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:43:19.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.168432 6787 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:43:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.168944 6787 kserver.cc:163] Server-wide thread pool size limit: 3276
06:43:19.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.235715 6787 rpc_server.cc:307] RPC server started. Bound to: 127.6.121.188:44129
06:43:19.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.235819 6863 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.121.188:44129 every 8 connection(s)
06:43:19.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.238967 6787 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/data/info.pb
06:43:19.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.243929 6864 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:43:19.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.249094 6630 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu as pid 6787
06:43:19.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.249408 6630 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/master-2/wal/instance
06:43:19.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.257633 6864 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } has no permanent_uuid. Determining permanent_uuid...
06:43:19.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.284466 6864 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } has no permanent_uuid. Determining permanent_uuid...
06:43:19.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.298344 6864 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } has no permanent_uuid. Determining permanent_uuid...
06:43:19.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.299111 6710 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } attempt: 6
06:43:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.320974 6864 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257: Bootstrap starting.
06:43:19.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.324227 6710 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299: Bootstrap starting.
06:43:19.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.329589 6864 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257: Neither blocks nor log segments found. Creating new log.
06:43:19.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.332103 6710 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299: Neither blocks nor log segments found. Creating new log.
06:43:19.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.332155 6864 log.cc:824] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257: Log is configured to *not* fsync() on all Append() calls
06:43:19.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.335338 6710 log.cc:824] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299: Log is configured to *not* fsync() on all Append() calls
06:43:19.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.339371 6864 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257: No bootstrap required, opened a new log
06:43:19.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.343276 6710 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299: No bootstrap required, opened a new log
06:43:19.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.371919 6864 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } }
06:43:19.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.372794 6864 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:43:19.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.373071 6864 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9787b799fe9e4859842e8e57eab53257, State: Initialized, Role: FOLLOWER
06:43:19.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.373968 6864 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } }
06:43:19.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.375404 6710 raft_consensus.cc:348] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } }
06:43:19.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.376592 6710 raft_consensus.cc:374] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:43:19.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.377084 6710 raft_consensus.cc:729] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27b6f55e7c514e47990ba697c9996299, State: Initialized, Role: FOLLOWER
06:43:19.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.376819 6876 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } } }
06:43:19.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.377629 6876 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [sys.catalog]: This master's current role is: FOLLOWER
06:43:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.378683 6864 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [sys.catalog]: configured and running, proceeding with master startup.
06:43:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.378516 6710 consensus_queue.cc:260] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } }
06:43:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.381233 6877 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } } }
06:43:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.382200 6877 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [sys.catalog]: This master's current role is: FOLLOWER
06:43:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.383598 6710 sys_catalog.cc:564] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [sys.catalog]: configured and running, proceeding with master startup.
06:43:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:19.394446 6664 tablet.cc:2369] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:43:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:19.423816 6818 tablet.cc:2369] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:43:19.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:19.426558 6896 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:19.426893 6896 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:19.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:19.439221 6899 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:19.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:19.439895 6899 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:19.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.410022 6787 master_runner.cc:186] Error getting master registration for 127.6.121.190:34235: OK, instance_id { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" instance_seqno: 1790664195685505 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:43:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.444307 6787 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:43:19.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.416113 6642 master_runner.cc:186] Error getting master registration for 127.6.121.189:46423: OK, instance_id { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" instance_seqno: 1790664197435161 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:43:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.463276 6642 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:43:19.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.771111 6877 raft_consensus.cc:482] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:43:19.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.771692 6877 raft_consensus.cc:504] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } }
06:43:19.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.778625 6839 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "27b6f55e7c514e47990ba697c9996299" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9787b799fe9e4859842e8e57eab53257" is_pre_election: true
06:43:19.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.779573 6839 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 27b6f55e7c514e47990ba697c9996299 in term 0.
06:43:19.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.781693 6877 leader_election.cc:288] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2a5e73069e934b468be51251ac8e357c (127.6.121.189:46423), 9787b799fe9e4859842e8e57eab53257 (127.6.121.188:44129)
06:43:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.780965 6662 leader_election.cc:302] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 27b6f55e7c514e47990ba697c9996299, 9787b799fe9e4859842e8e57eab53257; no voters:
06:43:19.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.784969 6877 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:43:19.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.785341 6877 raft_consensus.cc:482] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:43:19.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.785642 6877 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [term 0 FOLLOWER]: Advancing to term 1
06:43:19.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.792371 6877 raft_consensus.cc:504] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } }
06:43:19.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.794265 6877 leader_election.cc:288] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [CANDIDATE]: Term 1 election: Requested vote from peers 2a5e73069e934b468be51251ac8e357c (127.6.121.189:46423), 9787b799fe9e4859842e8e57eab53257 (127.6.121.188:44129)
06:43:19.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.795951 6839 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "27b6f55e7c514e47990ba697c9996299" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9787b799fe9e4859842e8e57eab53257"
06:43:19.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.796535 6839 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [term 0 FOLLOWER]: Advancing to term 1
06:43:19.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.803694 6839 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 27b6f55e7c514e47990ba697c9996299 in term 1.
06:43:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.804697 6662 leader_election.cc:302] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 27b6f55e7c514e47990ba697c9996299, 9787b799fe9e4859842e8e57eab53257; no voters:
06:43:19.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.805649 6877 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [term 1 FOLLOWER]: Leader election won for term 1
06:43:19.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.811100 6877 raft_consensus.cc:686] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [term 1 LEADER]: Becoming Leader. State: Replica: 27b6f55e7c514e47990ba697c9996299, State: Running, Role: LEADER
06:43:19.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.813047 6785 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } attempt: 7
06:43:19.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.812891 6877 consensus_queue.cc:237] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } }
06:43:19.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.837906 6785 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c: Bootstrap starting.
06:43:19.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.842468 6902 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 27b6f55e7c514e47990ba697c9996299. Latest consensus state: current_term: 1 leader_uuid: "27b6f55e7c514e47990ba697c9996299" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } } }
06:43:19.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.843276 6902 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [sys.catalog]: This master's current role is: LEADER
06:43:19.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.853149 6905 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
06:43:19.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.858793 6785 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c: Neither blocks nor log segments found. Creating new log.
06:43:19.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.860703 6905 catalog_manager.cc:1484] Initializing Kudu cluster ID...
06:43:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.864012 6785 log.cc:824] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c: Log is configured to *not* fsync() on all Append() calls
06:43:19.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.874341 6785 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c: No bootstrap required, opened a new log
06:43:19.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.898145 6839 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [term 1 FOLLOWER]: Refusing update from remote peer 27b6f55e7c514e47990ba697c9996299: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:43:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.899991 6877 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:43:19.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.930296 6785 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } }
06:43:19.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.934224 6785 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:43:19.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.935770 6785 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a5e73069e934b468be51251ac8e357c, State: Initialized, Role: FOLLOWER
06:43:19.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.939002 6785 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } }
06:43:19.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.948984 6908 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 27b6f55e7c514e47990ba697c9996299. Latest consensus state: current_term: 1 leader_uuid: "27b6f55e7c514e47990ba697c9996299" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } } }
06:43:19.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.950855 6908 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [sys.catalog]: This master's current role is: FOLLOWER
06:43:19.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.957121 6907 mvcc.cc:205] Tried to move back new op lower bound from 7334560562755743744 to 7334560562453266432. Current Snapshot: MvccSnapshot[applied={T|T < 7334560562755743744}]
06:43:19.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.958940 6909 mvcc.cc:205] Tried to move back new op lower bound from 7334560562755743744 to 7334560562453266432. Current Snapshot: MvccSnapshot[applied={T|T < 7334560562755743744}]
06:43:19.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.964783 6911 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } } }
06:43:19.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.967053 6911 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c [sys.catalog]: This master's current role is: FOLLOWER
06:43:19.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.972702 6908 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "27b6f55e7c514e47990ba697c9996299" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } } }
06:43:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.973462 6908 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257 [sys.catalog]: This master's current role is: FOLLOWER
06:43:19.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.975853 6902 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "27b6f55e7c514e47990ba697c9996299" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } } }
06:43:19.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.976629 6902 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [sys.catalog]: This master's current role is: LEADER
06:43:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.982514 6785 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c [sys.catalog]: configured and running, proceeding with master startup.
06:43:19.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.983584 6758 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c [term 0 FOLLOWER]: Advancing to term 1
06:43:19.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.983985 6759 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "27b6f55e7c514e47990ba697c9996299" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a5e73069e934b468be51251ac8e357c"
06:43:19.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.988274 6905 catalog_manager.cc:1347] Generated new cluster ID: b47ea6923c3b41a3be613cc93a4dbd02
06:43:19.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.988587 6905 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
06:43:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.984771 6760 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "27b6f55e7c514e47990ba697c9996299" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a5e73069e934b468be51251ac8e357c" is_pre_election: true
06:43:19.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.994187 6758 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c [term 1 FOLLOWER]: Refusing update from remote peer 27b6f55e7c514e47990ba697c9996299: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:43:19.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:19.995821 6902 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:43:20.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.031252 6905 catalog_manager.cc:1370] Generated new certificate authority record
06:43:20.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.034880 6905 catalog_manager.cc:1504] Loading token signing keys...
06:43:20.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.064772 6902 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "27b6f55e7c514e47990ba697c9996299" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } } }
06:43:20.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.065562 6902 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299 [sys.catalog]: This master's current role is: LEADER
06:43:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.068703 6918 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "27b6f55e7c514e47990ba697c9996299" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } } }
06:43:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.069672 6918 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c [sys.catalog]: This master's current role is: FOLLOWER
06:43:20.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.068645 6911 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 27b6f55e7c514e47990ba697c9996299. Latest consensus state: current_term: 1 leader_uuid: "27b6f55e7c514e47990ba697c9996299" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "27b6f55e7c514e47990ba697c9996299" member_type: VOTER last_known_addr { host: "127.6.121.190" port: 34235 } } peers { permanent_uuid: "2a5e73069e934b468be51251ac8e357c" member_type: VOTER last_known_addr { host: "127.6.121.189" port: 46423 } } peers { permanent_uuid: "9787b799fe9e4859842e8e57eab53257" member_type: VOTER last_known_addr { host: "127.6.121.188" port: 44129 } } }
06:43:20.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.072310 6911 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c [sys.catalog]: This master's current role is: FOLLOWER
06:43:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.083062 6905 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 27b6f55e7c514e47990ba697c9996299: Generated new TSK 0
06:43:20.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.086830 6905 catalog_manager.cc:1514] Initializing in-progress tserver states...
06:43:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.104241 6630 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:20.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/wal
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/data
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/logs
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/data/info.pb
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.121.129:0
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.121.129
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.121.129
06:43:20.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:20.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.121.190:34235,127.6.121.189:46423,127.6.121.188:44129
06:43:20.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.121.148:45515
06:43:20.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:20.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:20.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:43:20.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.126096 6929 catalog_manager.cc:1259] Loaded cluster ID: b47ea6923c3b41a3be613cc93a4dbd02
06:43:20.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.126706 6929 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c: loading cluster ID for follower catalog manager: success
06:43:20.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.134754 6929 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c: acquiring CA information for follower catalog manager: success
06:43:20.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.142077 6929 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 2a5e73069e934b468be51251ac8e357c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:43:20.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.432337 6896 catalog_manager.cc:1259] Loaded cluster ID: b47ea6923c3b41a3be613cc93a4dbd02
06:43:20.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.432958 6896 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257: loading cluster ID for follower catalog manager: success
06:43:20.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.440869 6896 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257: acquiring CA information for follower catalog manager: success
06:43:20.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.456233 6896 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 9787b799fe9e4859842e8e57eab53257: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:43:20.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:20.716964 6931 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:43:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:20.717940 6931 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:43:20.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:20.718879 6931 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:43:20.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:20.772701 6931 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:43:20.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:20.774295 6931 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.121.129
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.832675 6931 tablet_server_runner.cc:78] Tablet server non-default flags:
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.121.148:45515
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/data
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/wal
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.121.129:0
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/data/info.pb
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.121.129
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:20.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.121.190:34235,127.6.121.189:46423,127.6.121.188:44129
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.121.129
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/logs
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:43:20.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:43:20.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.835856 6931 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:43:20.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:20.838214 6931 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:43:20.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:20.880054 6937 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:23.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:20.881567 6938 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:23.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:22.266261 6936 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 6931
06:43:23.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:23.388315 6936 kernel_stack_watchdog.cc:198] Thread 6931 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 382ms:
06:43:23.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
06:43:23.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
06:43:23.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
06:43:23.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
06:43:23.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:23.388429 6931 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.506s user 0.001s sys 0.000s
06:43:23.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:23.388806 6931 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.507s user 0.001s sys 0.000s
06:43:23.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:23.391515 6941 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:23.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:23.395085 6939 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2501 milliseconds
06:43:23.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.395113 6931 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:43:23.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.396615 6931 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:43:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.399153 6931 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:43:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.400578 6931 hybrid_clock.cc:648] HybridClock initialized: now 1790664203400542 us; error 35 us; skew 500 ppm
06:43:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.401574 6931 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:43:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.415678 6931 webserver.cc:460] Webserver started at http://127.6.121.129:45725/ using document root <none> and password file <none>
06:43:23.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.419411 6931 fs_manager.cc:359] Metadata directory not provided
06:43:23.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.419668 6931 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:43:23.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.420197 6931 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:43:23.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.424995 6931 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/data/instance:
06:43:23.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9395e3814ed64359ba66066540384340"
06:43:23.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:23 on dist-test-slave-vcpq"
06:43:23.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.426177 6931 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/wal/instance:
06:43:23.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9395e3814ed64359ba66066540384340"
06:43:23.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:23 on dist-test-slave-vcpq"
06:43:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.433645 6931 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
06:43:23.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.440008 6947 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:43:23.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.441171 6931 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
06:43:23.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.441452 6931 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/data,/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/wal
06:43:23.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9395e3814ed64359ba66066540384340"
06:43:23.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:23 on dist-test-slave-vcpq"
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.441740 6931 fs_report.cc:383] FS layout report
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/wal
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/wal
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/data/data
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:43:23.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:43:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:43:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.483634 6931 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:43:23.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.485388 6931 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:43:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.485922 6931 kserver.cc:163] Server-wide thread pool size limit: 3276
06:43:23.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.489185 6931 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:43:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.494143 6931 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:43:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.494380 6931 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:43:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.494668 6931 ts_tablet_manager.cc:610] Registered 0 tablets
06:43:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.494947 6931 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
06:43:23.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.633699 6931 rpc_server.cc:307] RPC server started. Bound to: 127.6.121.129:42019
06:43:23.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.633817 7059 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.121.129:42019 every 8 connection(s)
06:43:23.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.636255 6931 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/data/info.pb
06:43:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.643729 6630 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu as pid 6931
06:43:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.644294 6630 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-0/wal/instance
06:43:23.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.670064 6630 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/wal
06:43:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/data
06:43:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:43:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:43:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:43:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/logs
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/data/info.pb
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.121.130:0
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.121.130
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.121.130
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.121.190:34235,127.6.121.189:46423,127.6.121.188:44129
06:43:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.121.148:45515
06:43:23.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:23.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:23.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:43:23.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.699896 7061 heartbeater.cc:347] Connected to a master server at 127.6.121.189:46423
06:43:23.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.700407 7061 heartbeater.cc:464] Registering TS with master...
06:43:23.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.701663 7061 heartbeater.cc:511] Master 127.6.121.189:46423 requested a full tablet report, sending...
06:43:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.704923 6750 ts_manager.cc:194] Registered new tserver with Master: 9395e3814ed64359ba66066540384340 (127.6.121.129:42019)
06:43:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.706022 7062 heartbeater.cc:347] Connected to a master server at 127.6.121.190:34235
06:43:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.706342 7062 heartbeater.cc:464] Registering TS with master...
06:43:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.707118 7062 heartbeater.cc:511] Master 127.6.121.190:34235 requested a full tablet report, sending...
06:43:23.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.710341 6675 ts_manager.cc:194] Registered new tserver with Master: 9395e3814ed64359ba66066540384340 (127.6.121.129:42019)
06:43:23.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.713177 6675 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.121.129:60543
06:43:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.713971 7060 heartbeater.cc:347] Connected to a master server at 127.6.121.188:44129
06:43:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.714303 7060 heartbeater.cc:464] Registering TS with master...
06:43:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.715093 7060 heartbeater.cc:511] Master 127.6.121.188:44129 requested a full tablet report, sending...
06:43:23.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:23.719079 6829 ts_manager.cc:194] Registered new tserver with Master: 9395e3814ed64359ba66066540384340 (127.6.121.129:42019)
06:43:23.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:23.990491 7070 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:43:23.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:23.991104 7070 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:43:23.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:23.991714 7070 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:43:24.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:24.022512 7070 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:43:24.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:24.023537 7070 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.121.130
06:43:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:24.056586 7070 tablet_server_runner.cc:78] Tablet server non-default flags:
06:43:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.121.148:45515
06:43:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/data
06:43:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/wal
06:43:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.121.130:0
06:43:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:43:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:24.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/data/info.pb
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.121.130
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.121.190:34235,127.6.121.189:46423,127.6.121.188:44129
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.121.130
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/logs
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:43:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:43:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:43:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:43:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:43:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:24.058126 7070 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:43:24.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:24.060423 7070 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:43:24.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:24.079715 7078 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:24.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:24.717873 7062 heartbeater.cc:503] Master 127.6.121.190:34235 was elected leader, sending a full tablet report...
06:43:25.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:24.079816 7077 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:25.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:24.081118 7080 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:25.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:25.379880 7079 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:43:25.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.379930 7070 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:43:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.383865 7070 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:43:25.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.386720 7070 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:43:25.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.388175 7070 hybrid_clock.cc:648] HybridClock initialized: now 1790664205388115 us; error 79 us; skew 500 ppm
06:43:25.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.389245 7070 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:43:25.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.397375 7070 webserver.cc:460] Webserver started at http://127.6.121.130:34939/ using document root <none> and password file <none>
06:43:25.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.398240 7070 fs_manager.cc:359] Metadata directory not provided
06:43:25.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.398429 7070 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:43:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.398904 7070 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:43:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.403678 7070 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/data/instance:
06:43:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8e17c5a2472431ea2e927341fb08f21"
06:43:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:25 on dist-test-slave-vcpq"
06:43:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.404695 7070 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/wal/instance:
06:43:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8e17c5a2472431ea2e927341fb08f21"
06:43:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:25 on dist-test-slave-vcpq"
06:43:25.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.411594 7070 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
06:43:25.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.417003 7088 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:43:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.417987 7070 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.004s
06:43:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.418262 7070 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/data,/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/wal
06:43:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8e17c5a2472431ea2e927341fb08f21"
06:43:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:25 on dist-test-slave-vcpq"
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.418551 7070 fs_report.cc:383] FS layout report
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/wal
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/wal
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/data/data
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:43:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:43:25.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.454370 7070 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:43:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.456121 7070 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:43:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.456665 7070 kserver.cc:163] Server-wide thread pool size limit: 3276
06:43:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.460388 7070 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:43:25.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.466064 7070 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:43:25.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.466351 7070 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:43:25.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.466624 7070 ts_tablet_manager.cc:610] Registered 0 tablets
06:43:25.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.466840 7070 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
06:43:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.606315 7070 rpc_server.cc:307] RPC server started. Bound to: 127.6.121.130:36391
06:43:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.606421 7200 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.121.130:36391 every 8 connection(s)
06:43:25.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.608851 7070 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/data/info.pb
06:43:25.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.612215 6630 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu as pid 7070
06:43:25.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.612836 6630 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-1/wal/instance
06:43:25.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.624218 6630 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:25.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:25.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/wal
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/data
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/logs
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/data/info.pb
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.121.131:0
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.121.131
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.121.131
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.121.190:34235,127.6.121.189:46423,127.6.121.188:44129
06:43:25.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.121.148:45515
06:43:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:43:25.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.683614 7202 heartbeater.cc:347] Connected to a master server at 127.6.121.189:46423
06:43:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.684100 7202 heartbeater.cc:464] Registering TS with master...
06:43:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.685308 7202 heartbeater.cc:511] Master 127.6.121.189:46423 requested a full tablet report, sending...
06:43:25.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.687626 7203 heartbeater.cc:347] Connected to a master server at 127.6.121.190:34235
06:43:25.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.687919 7203 heartbeater.cc:464] Registering TS with master...
06:43:25.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.688522 6750 ts_manager.cc:194] Registered new tserver with Master: a8e17c5a2472431ea2e927341fb08f21 (127.6.121.130:36391)
06:43:25.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.688686 7203 heartbeater.cc:511] Master 127.6.121.190:34235 requested a full tablet report, sending...
06:43:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.690686 7201 heartbeater.cc:347] Connected to a master server at 127.6.121.188:44129
06:43:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.691068 7201 heartbeater.cc:464] Registering TS with master...
06:43:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.691784 7201 heartbeater.cc:511] Master 127.6.121.188:44129 requested a full tablet report, sending...
06:43:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.691723 6675 ts_manager.cc:194] Registered new tserver with Master: a8e17c5a2472431ea2e927341fb08f21 (127.6.121.130:36391)
06:43:25.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.693730 6675 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.121.130:36143
06:43:25.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:25.693997 6829 ts_manager.cc:194] Registered new tserver with Master: a8e17c5a2472431ea2e927341fb08f21 (127.6.121.130:36391)
06:43:25.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:25.972674 7211 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:43:25.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:25.973206 7211 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:43:25.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:25.973841 7211 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:43:26.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:26.006479 7211 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:43:26.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:26.007462 7211 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.121.131
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:26.042587 7211 tablet_server_runner.cc:78] Tablet server non-default flags:
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.121.148:45515
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/data
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/wal
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.121.131:0
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/data/info.pb
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.121.131
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.121.190:34235,127.6.121.189:46423,127.6.121.188:44129
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.121.131
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/logs
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:43:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:43:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:26.043793 7211 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:43:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:26.045564 7211 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:43:26.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:26.064931 7219 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:26.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:26.697805 7203 heartbeater.cc:503] Master 127.6.121.190:34235 was elected leader, sending a full tablet report...
06:43:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:26.065577 7218 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:26.066349 7211 server_base.cc:1034] running on GCE node
06:43:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:26.066819 7221 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:27.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.350504 7211 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:43:27.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.352770 7211 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:43:27.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.354239 7211 hybrid_clock.cc:648] HybridClock initialized: now 1790664207354104 us; error 61 us; skew 500 ppm
06:43:27.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.355161 7211 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:43:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.361783 7211 webserver.cc:460] Webserver started at http://127.6.121.131:43503/ using document root <none> and password file <none>
06:43:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.362897 7211 fs_manager.cc:359] Metadata directory not provided
06:43:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.363144 7211 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:43:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.363729 7211 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:43:27.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.368546 7211 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/data/instance:
06:43:27.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9013186afe2a4ed98c3419dcb11d1ef1"
06:43:27.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:27 on dist-test-slave-vcpq"
06:43:27.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.369714 7211 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/wal/instance:
06:43:27.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9013186afe2a4ed98c3419dcb11d1ef1"
06:43:27.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:27 on dist-test-slave-vcpq"
06:43:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.377032 7211 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
06:43:27.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.382730 7228 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:43:27.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.383755 7211 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
06:43:27.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.384095 7211 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/data,/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/wal
06:43:27.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9013186afe2a4ed98c3419dcb11d1ef1"
06:43:27.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:27 on dist-test-slave-vcpq"
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.384471 7211 fs_report.cc:383] FS layout report
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/wal
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/wal
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/data/data
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:43:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:43:27.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.423403 7211 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:43:27.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.424938 7211 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:43:27.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.425448 7211 kserver.cc:163] Server-wide thread pool size limit: 3276
06:43:27.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.427946 7211 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:43:27.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.431962 7211 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:43:27.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.432190 7211 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:43:27.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.432544 7211 ts_tablet_manager.cc:610] Registered 0 tablets
06:43:27.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.432729 7211 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:43:27.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.577076 7211 rpc_server.cc:307] RPC server started. Bound to: 127.6.121.131:40833
06:43:27.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.577140 7340 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.121.131:40833 every 8 connection(s)
06:43:27.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.579747 7211 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/data/info.pb
06:43:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.588194 6630 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu as pid 7211
06:43:27.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.588740 6630 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster4644700589948767112/ts-2/wal/instance
06:43:27.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.622555 7342 heartbeater.cc:347] Connected to a master server at 127.6.121.189:46423
06:43:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.623081 7342 heartbeater.cc:464] Registering TS with master...
06:43:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.623870 7343 heartbeater.cc:347] Connected to a master server at 127.6.121.190:34235
06:43:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.624214 7343 heartbeater.cc:464] Registering TS with master...
06:43:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.624243 7342 heartbeater.cc:511] Master 127.6.121.189:46423 requested a full tablet report, sending...
06:43:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.624424 7341 heartbeater.cc:347] Connected to a master server at 127.6.121.188:44129
06:43:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.624747 7341 heartbeater.cc:464] Registering TS with master...
06:43:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.625150 7343 heartbeater.cc:511] Master 127.6.121.190:34235 requested a full tablet report, sending...
06:43:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.625731 7341 heartbeater.cc:511] Master 127.6.121.188:44129 requested a full tablet report, sending...
06:43:27.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.627719 6675 ts_manager.cc:194] Registered new tserver with Master: 9013186afe2a4ed98c3419dcb11d1ef1 (127.6.121.131:40833)
06:43:27.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.628192 6829 ts_manager.cc:194] Registered new tserver with Master: 9013186afe2a4ed98c3419dcb11d1ef1 (127.6.121.131:40833)
06:43:27.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.628955 6749 ts_manager.cc:194] Registered new tserver with Master: 9013186afe2a4ed98c3419dcb11d1ef1 (127.6.121.131:40833)
06:43:27.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.629611 6675 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.121.131:52975
06:43:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:27.638460 6630 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
06:43:27.649 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:43:27.650 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
06:43:27.661 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.6.121.190"
port: 34235
}
bound_http_address {
host: "127.6.121.190"
port: 36423
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.6.121.189"
port: 46423
}
bound_http_address {
host: "127.6.121.189"
port: 35473
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.6.121.188"
port: 44129
}
bound_http_address {
host: "127.6.121.188"
port: 46461
}
}
}
06:43:27.667 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
06:43:27.669 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.6.121.129"
port: 42019
}
bound_http_address {
host: "127.6.121.129"
port: 45725
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.6.121.130"
port: 36391
}
bound_http_address {
host: "127.6.121.130"
port: 34939
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.6.121.131"
port: 40833
}
bound_http_address {
host: "127.6.121.131"
port: 43503
}
}
}
06:43:27.670 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
06:43:27.854 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
06:43:27.957 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskUfq9I_/test-tmp
06:43:27.958 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
06:43:27.959 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskUfq9I_/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
06:43:27.961 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
06:43:28.294 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:43:28.294 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
06:43:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:28.297956 7354 mini_oidc.cc:152] Starting JWKS server
06:43:28.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:28.301362 7354 webserver.cc:460] Webserver started at http://127.0.0.1:42693/ using document root <none> and password file <none>
06:43:28.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:28.301764 7354 mini_oidc.cc:189] Starting OIDC Discovery server
06:43:28.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:28.303503 7354 webserver.cc:460] Webserver started at http://127.0.0.1:43223/ using document root <none> and password file <none>
06:43:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
06:43:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/principal' for realm 'KRBTEST.COM',
06:43:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
06:43:28.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:28 dist-test-slave-vcpq krb5kdc[7366](info): setting up network...
06:43:28.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
06:43:28.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:28 dist-test-slave-vcpq krb5kdc[7366](info): set up 2 sockets
06:43:28.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:28 dist-test-slave-vcpq krb5kdc[7366](info): commencing operation
06:43:28.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
06:43:28.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:28.633898 7343 heartbeater.cc:503] Master 127.6.121.190:34235 was elected leader, sending a full tablet report...
06:43:29.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:29.444736 7354 mini_kdc.cc:121] Time spent starting KDC: real 1.117s user 0.000s sys 0.019s
06:43:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
06:43:29.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:43:29.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
06:43:29.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
06:43:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:43:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
06:43:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
06:43:29.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:43:29.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
06:43:29.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:43:29.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/test-user.keytab.
06:43:29.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/test-user.keytab.
06:43:29.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:29 dist-test-slave-vcpq krb5kdc[7366](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664209, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:43:29.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:43:29.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T06:43:29Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:43:29.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T06:43:29Z Disabled control of system clock
06:43:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.46.190@KRBTEST.COM; defaulting to no policy
06:43:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:43:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.46.190@KRBTEST.COM" created.
06:43:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:43:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.46.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.190.keytab.
06:43:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.46.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.190.keytab.
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:29.643899 7354 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/logs
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data/info.pb
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.46.190:32891
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.46.190
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.190.keytab
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.46.190
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.46.148:42101
06:43:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:29.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:29.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:29.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:29.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.46.190:32891,127.7.46.189:45741,127.7.46.188:43659
06:43:29.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:29.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42693/jwks.json
06:43:29.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:43:29.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:29.953649 7379 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:43:29.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:29.954351 7379 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:43:29.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:29.955070 7379 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:43:29.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:29.986634 7379 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:43:29.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:29.986963 7379 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:43:29.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:29.987344 7379 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:43:29.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:29.987653 7379 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:43:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:29.987979 7379 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:43:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:29.988303 7379 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:43:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:29.988549 7379 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42693/jwks.json
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:30.028683 7379 master_runner.cc:386] Master server non-default flags:
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.46.148:42101
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.46.190:32891,127.7.46.189:45741,127.7.46.188:43659
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.190.keytab
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.46.190
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.46.190:32891
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42693/jwks.json
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data/info.pb
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.46.190
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/logs
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:43:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:43:30.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:43:30.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:43:30.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:43:30.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:30.029763 7379 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:43:30.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:30.030709 7379 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:43:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:30.049477 7379 server_base.cc:1034] running on GCE node
06:43:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:30.047324 7385 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:30.047281 7388 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:30.047295 7386 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.322347 7379 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:43:31.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.324882 7379 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:43:31.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.326221 7379 hybrid_clock.cc:648] HybridClock initialized: now 1790664211326175 us; error 63 us; skew 500 ppm
06:43:31.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.327044 7379 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:43:31.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:31 dist-test-slave-vcpq krb5kdc[7366](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664211, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:43:31.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.333232 7379 init.cc:377] Logged in from keytab as kudu/127.7.46.190@KRBTEST.COM (short username kudu)
06:43:31.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.338722 7379 webserver.cc:460] Webserver started at http://127.7.46.190:44603/ using document root <none> and password file <none>
06:43:31.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.339673 7379 fs_manager.cc:359] Metadata directory not provided
06:43:31.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.339855 7379 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:43:31.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.340250 7379 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:43:31.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.345270 7379 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data/instance:
06:43:31.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd9fd47cf55245a4bfe60cac955c07c1"
06:43:31.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:31 on dist-test-slave-vcpq"
06:43:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.346297 7379 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal/instance:
06:43:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd9fd47cf55245a4bfe60cac955c07c1"
06:43:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:31 on dist-test-slave-vcpq"
06:43:31.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.353245 7379 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
06:43:31.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.358780 7397 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:43:31.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.359912 7379 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:43:31.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.360203 7379 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data,/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal
06:43:31.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd9fd47cf55245a4bfe60cac955c07c1"
06:43:31.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:31 on dist-test-slave-vcpq"
06:43:31.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.360505 7379 fs_report.cc:383] FS layout report
06:43:31.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:43:31.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal
06:43:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal
06:43:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data/data
06:43:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:43:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:43:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:43:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:43:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:43:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:43:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:43:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:43:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:43:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:43:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:43:31.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.406929 7379 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:43:31.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.412876 7379 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:43:31.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.413286 7379 kserver.cc:163] Server-wide thread pool size limit: 3276
06:43:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.491443 7379 rpc_server.cc:307] RPC server started. Bound to: 127.7.46.190:32891
06:43:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.491557 7448 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.46.190:32891 every 8 connection(s)
06:43:31.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.494141 7379 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data/info.pb
06:43:31.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.497609 7354 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu as pid 7379
06:43:31.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.498176 7354 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal/instance
06:43:31.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.500313 7449 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:43:31.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.514500 7449 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } has no permanent_uuid. Determining permanent_uuid...
06:43:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.46.189@KRBTEST.COM; defaulting to no policy
06:43:31.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:43:31.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.46.189@KRBTEST.COM" created.
06:43:31.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:31 dist-test-slave-vcpq krb5kdc[7366](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664211, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.190@KRBTEST.COM for kudu/127.7.46.190@KRBTEST.COM
06:43:31.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:43:31.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.46.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.189.keytab.
06:43:31.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.46.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.189.keytab.
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.545157 7354 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/logs
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data/info.pb
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:43:31.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.46.189:45741
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.46.189
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.189.keytab
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.46.189
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.46.148:42101
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.46.190:32891,127.7.46.189:45741,127.7.46.188:43659
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42693/jwks.json
06:43:31.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:43:31.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.552466 7449 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } has no permanent_uuid. Determining permanent_uuid...
06:43:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.555126 7398 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:31.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.559628 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.189:45741: Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:31.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.609161 7449 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } attempt: 1
06:43:31.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.613161 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.189:45741: Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:31.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.681574 7449 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } attempt: 2
06:43:31.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.685402 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.189:45741: Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:31.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.776813 7449 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } attempt: 3
06:43:31.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.780802 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.189:45741: Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:31.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.873735 7455 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:43:31.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.874401 7455 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:43:31.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.875140 7455 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:43:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.905938 7455 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:43:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.906219 7455 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:43:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.906643 7455 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:43:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.906980 7455 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:43:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.907282 7455 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:43:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.907604 7455 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:43:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.907829 7455 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42693/jwks.json
06:43:31.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.924152 7449 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } attempt: 4
06:43:31.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.927613 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.189:45741: Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.944867 7455 master_runner.cc:386] Master server non-default flags:
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.46.148:42101
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.46.190:32891,127.7.46.189:45741,127.7.46.188:43659
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.189.keytab
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.46.189
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.46.189:45741
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42693/jwks.json
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data/info.pb
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.46.189
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/logs
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:43:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:43:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:43:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:43:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:43:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:43:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.946383 7455 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:43:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:31.947763 7455 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:43:31.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.967113 7462 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:32.226979 7449 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } attempt: 5
06:43:32.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:32.231089 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.189:45741: Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:32.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:32.778720 7449 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } attempt: 6
06:43:32.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:32.785315 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.189:45741: Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:33.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.968056 7465 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:33.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:31.968676 7463 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.191754 7464 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:43:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.191793 7455 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:43:33.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.196018 7455 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:43:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.198549 7455 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:43:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.199891 7455 hybrid_clock.cc:648] HybridClock initialized: now 1790664213199843 us; error 82 us; skew 500 ppm
06:43:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.200639 7455 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:43:33.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:33 dist-test-slave-vcpq krb5kdc[7366](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664213, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:43:33.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.206791 7455 init.cc:377] Logged in from keytab as kudu/127.7.46.189@KRBTEST.COM (short username kudu)
06:43:33.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.211519 7455 webserver.cc:460] Webserver started at http://127.7.46.189:44729/ using document root <none> and password file <none>
06:43:33.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.212357 7455 fs_manager.cc:359] Metadata directory not provided
06:43:33.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.212529 7455 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:43:33.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.212901 7455 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:43:33.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.217583 7455 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data/instance:
06:43:33.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9fc30b95b2748bcab06730f45e93eae"
06:43:33.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:33 on dist-test-slave-vcpq"
06:43:33.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.218683 7455 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal/instance:
06:43:33.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9fc30b95b2748bcab06730f45e93eae"
06:43:33.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:33 on dist-test-slave-vcpq"
06:43:33.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.225694 7455 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
06:43:33.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.230822 7475 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:43:33.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.231985 7455 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:43:33.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.232371 7455 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data,/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal
06:43:33.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9fc30b95b2748bcab06730f45e93eae"
06:43:33.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:33 on dist-test-slave-vcpq"
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.232656 7455 fs_report.cc:383] FS layout report
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data/data
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:43:33.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:43:33.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.301373 7455 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:43:33.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.307232 7455 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:43:33.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.307622 7455 kserver.cc:163] Server-wide thread pool size limit: 3276
06:43:33.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.377125 7455 rpc_server.cc:307] RPC server started. Bound to: 127.7.46.189:45741
06:43:33.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.377208 7526 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.46.189:45741 every 8 connection(s)
06:43:33.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.380043 7455 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data/info.pb
06:43:33.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.384948 7527 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:43:33.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.386514 7354 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu as pid 7455
06:43:33.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.386894 7354 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal/instance
06:43:33.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.398996 7527 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } has no permanent_uuid. Determining permanent_uuid...
06:43:33.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.7.46.188@KRBTEST.COM; defaulting to no policy
06:43:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:43:33.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.7.46.188@KRBTEST.COM" created.
06:43:33.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:33 dist-test-slave-vcpq krb5kdc[7366](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664213, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.189@KRBTEST.COM for kudu/127.7.46.190@KRBTEST.COM
06:43:33.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:43:33.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.46.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.188.keytab.
06:43:33.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.7.46.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.188.keytab.
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.435737 7354 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/wal
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/logs
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data/info.pb
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:33.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.46.188:43659
06:43:33.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.46.188
06:43:33.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:33.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.188.keytab
06:43:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.46.188
06:43:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:43:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:43:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:43:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.46.148:42101
06:43:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.46.190:32891,127.7.46.189:45741,127.7.46.188:43659
06:43:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42693/jwks.json
06:43:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:43:33.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.441027 7527 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } has no permanent_uuid. Determining permanent_uuid...
06:43:33.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:33 dist-test-slave-vcpq krb5kdc[7366](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664213, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.189@KRBTEST.COM for kudu/127.7.46.189@KRBTEST.COM
06:43:33.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.481833 7527 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } has no permanent_uuid. Determining permanent_uuid...
06:43:33.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.484223 7478 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:33.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.489280 7527 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:33.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.538803 7527 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } attempt: 1
06:43:33.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.542276 7527 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:33.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.610714 7527 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } attempt: 2
06:43:33.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.614907 7527 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:33.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.706251 7527 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } attempt: 3
06:43:33.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.710669 7527 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.779021 7533 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:43:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.779726 7533 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:43:33.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.780510 7533 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:43:33.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.812960 7533 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:43:33.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.813236 7533 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:43:33.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.813634 7533 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:43:33.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.813967 7533 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:43:33.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.814281 7533 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:43:33.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.814584 7533 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:43:33.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.814844 7533 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42693/jwks.json
06:43:33.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.845956 7449 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } attempt: 7
06:43:33.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.853727 7533 master_runner.cc:386] Master server non-default flags:
06:43:33.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:33.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.46.148:42101
06:43:33.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:33.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:33.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data
06:43:33.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/wal
06:43:33.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:43:33.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:33.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.46.190:32891,127.7.46.189:45741,127.7.46.188:43659
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.188.keytab
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.46.188
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.46.188:43659
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42693/jwks.json
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data/info.pb
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.46.188
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/logs
06:43:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:43:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:43:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:43:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:43:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:43:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:43:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:43:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:43:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:43:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.854099 7527 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } attempt: 4
06:43:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.855226 7533 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:43:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.856583 7533 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:43:33.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.858294 7527 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:33.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:33 dist-test-slave-vcpq krb5kdc[7366](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664211, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.190@KRBTEST.COM for kudu/127.7.46.189@KRBTEST.COM
06:43:33.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.881161 7543 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:33.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.882730 7449 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } has no permanent_uuid. Determining permanent_uuid...
06:43:33.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.887667 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:33.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.946076 7449 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } attempt: 1
06:43:33.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.950501 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:34.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:34.031868 7449 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } attempt: 2
06:43:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:34.035637 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:34.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:34.121076 7449 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } attempt: 3
06:43:34.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:34.125228 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:34.157696 7527 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } attempt: 5
06:43:34.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:34.161850 7527 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:34.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:34.265588 7449 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } attempt: 4
06:43:34.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:34.268914 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:34.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:34.552305 7449 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } attempt: 5
06:43:34.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:34.556238 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:34.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:34.709425 7527 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } attempt: 6
06:43:34.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:34.716102 7527 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:35.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.108875 7449 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } attempt: 6
06:43:35.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:35.115486 7449 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.7.46.188:43659: Network error: Client connection negotiation failed: client connection to 127.7.46.188:43659: connect: Connection refused (error 111)
06:43:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.881191 7544 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:33.881784 7546 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:33.881523 7533 server_base.cc:1034] running on GCE node
06:43:35.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.201660 7533 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:43:35.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.204648 7533 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:43:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.206074 7533 hybrid_clock.cc:648] HybridClock initialized: now 1790664215206016 us; error 77 us; skew 500 ppm
06:43:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.206979 7533 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:43:35.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:35 dist-test-slave-vcpq krb5kdc[7366](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664215, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:43:35.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.212917 7533 init.cc:377] Logged in from keytab as kudu/127.7.46.188@KRBTEST.COM (short username kudu)
06:43:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.217649 7533 webserver.cc:460] Webserver started at http://127.7.46.188:40401/ using document root <none> and password file <none>
06:43:35.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.218600 7533 fs_manager.cc:359] Metadata directory not provided
06:43:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.218883 7533 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:43:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.219285 7533 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:43:35.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.224352 7533 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data/instance:
06:43:35.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c2ec393d5534ef0a3398545ee67cdb8"
06:43:35.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:35 on dist-test-slave-vcpq"
06:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.225437 7533 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/wal/instance:
06:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c2ec393d5534ef0a3398545ee67cdb8"
06:43:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:35 on dist-test-slave-vcpq"
06:43:35.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.232463 7533 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.008s sys 0.000s
06:43:35.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.238422 7562 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:43:35.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.239645 7533 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
06:43:35.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.240011 7533 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data,/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/wal
06:43:35.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c2ec393d5534ef0a3398545ee67cdb8"
06:43:35.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:35 on dist-test-slave-vcpq"
06:43:35.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.240396 7533 fs_report.cc:383] FS layout report
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/wal
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/wal
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data/data
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:43:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:43:35.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.287628 7533 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:43:35.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.293088 7533 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:43:35.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.293498 7533 kserver.cc:163] Server-wide thread pool size limit: 3276
06:43:35.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.363780 7533 rpc_server.cc:307] RPC server started. Bound to: 127.7.46.188:43659
06:43:35.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.363904 7613 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.46.188:43659 every 8 connection(s)
06:43:35.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.366653 7533 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data/info.pb
06:43:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.371366 7354 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu as pid 7533
06:43:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.371853 7354 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/wal/instance
06:43:35.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.373347 7614 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:43:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.395639 7614 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } has no permanent_uuid. Determining permanent_uuid...
06:43:35.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:35 dist-test-slave-vcpq krb5kdc[7366](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664215, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.188@KRBTEST.COM for kudu/127.7.46.190@KRBTEST.COM
06:43:35.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:35 dist-test-slave-vcpq krb5kdc[7366](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664209, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.46.188@KRBTEST.COM
06:43:35.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.436587 7614 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } has no permanent_uuid. Determining permanent_uuid...
06:43:35.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:35 dist-test-slave-vcpq krb5kdc[7366](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664215, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.188@KRBTEST.COM for kudu/127.7.46.189@KRBTEST.COM
06:43:35.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.462777 7614 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } has no permanent_uuid. Determining permanent_uuid...
06:43:35.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:35 dist-test-slave-vcpq krb5kdc[7366](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664209, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.46.189@KRBTEST.COM
06:43:35.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:35 dist-test-slave-vcpq krb5kdc[7366](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664215, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.188@KRBTEST.COM for kudu/127.7.46.188@KRBTEST.COM
06:43:35.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:35 dist-test-slave-vcpq krb5kdc[7366](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664209, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.46.190@KRBTEST.COM
06:43:35.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.495486 7614 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: Bootstrap starting.
06:43:35.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.501430 7614 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: Neither blocks nor log segments found. Creating new log.
06:43:35.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.503706 7614 log.cc:824] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: Log is configured to *not* fsync() on all Append() calls
06:43:35.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.509316 7614 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: No bootstrap required, opened a new log
06:43:35.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.514985 7354 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
06:43:35.521 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:43:35.523 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
06:43:35.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.534327 7614 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd9fd47cf55245a4bfe60cac955c07c1" member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } } peers { permanent_uuid: "b9fc30b95b2748bcab06730f45e93eae" member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } } peers { permanent_uuid: "9c2ec393d5534ef0a3398545ee67cdb8" member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } }
06:43:35.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.535192 7614 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:43:35.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.535511 7614 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c2ec393d5534ef0a3398545ee67cdb8, State: Initialized, Role: FOLLOWER
06:43:35.536 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
06:43:35.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.536409 7614 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd9fd47cf55245a4bfe60cac955c07c1" member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } } peers { permanent_uuid: "b9fc30b95b2748bcab06730f45e93eae" member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } } peers { permanent_uuid: "9c2ec393d5534ef0a3398545ee67cdb8" member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } }
06:43:35.537 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
06:43:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.539369 7623 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd9fd47cf55245a4bfe60cac955c07c1" member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } } peers { permanent_uuid: "b9fc30b95b2748bcab06730f45e93eae" member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } } peers { permanent_uuid: "9c2ec393d5534ef0a3398545ee67cdb8" member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } } }
06:43:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.540347 7623 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [sys.catalog]: This master's current role is: FOLLOWER
06:43:35.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.541178 7614 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [sys.catalog]: configured and running, proceeding with master startup.
06:43:35.542 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.7.46.190"
port: 32891
}
bound_http_address {
host: "127.7.46.190"
port: 44603
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.7.46.189"
port: 45741
}
bound_http_address {
host: "127.7.46.189"
port: 44729
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.7.46.188"
port: 43659
}
bound_http_address {
host: "127.7.46.188"
port: 40401
}
}
}
06:43:35.542 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
06:43:35.543 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:43:35.543 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
06:43:35.545 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
06:43:35.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:35.549032 7568 tablet.cc:2369] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:43:35.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:35.572922 7635 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:35.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:35.573254 7635 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:35.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:35 dist-test-slave-vcpq krb5kdc[7366](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664215, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:43:35.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:43:35.593 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:43:35.596 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5cc
06:43:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.565683 7533 master_runner.cc:186] Error getting master registration for 127.7.46.190:32891: OK, instance_id { permanent_uuid: "bd9fd47cf55245a4bfe60cac955c07c1" instance_seqno: 1790664211468883 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:43:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.598575 7533 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:43:35.602 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:43:35.604 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5cc
06:43:35.605 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:43:35.609 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
06:43:35.610 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
06:43:35.611 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
06:43:35.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:35 dist-test-slave-vcpq krb5kdc[7366](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664215, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:43:35.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:43:35.645 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:43:35.645 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5cc
06:43:35.648 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:43:35.648 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.46.188:43659
06:43:35.650 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
06:43:35.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.651248 7354 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu with pid 7533
06:43:35.674 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:43:35.674 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.46.189:45741
06:43:35.675 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
06:43:35.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.675575 7354 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu with pid 7455
06:43:35.692 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:43:35.692 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.7.46.190:32891
06:43:35.693 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
06:43:35.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.693456 7354 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu with pid 7379
06:43:35.714 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:43:35.714 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.46.188:43659
06:43:35.716 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:35.717811 7354 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/wal
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/logs
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data/info.pb
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.46.188:43659
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.46.188
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40401
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.188.keytab
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.46.188
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.46.148:42101
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.46.190:32891,127.7.46.189:45741,127.7.46.188:43659
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42693/jwks.json
06:43:35.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:43:36.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:36.034850 7640 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:43:36.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:36.035529 7640 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:43:36.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:36.036386 7640 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:43:36.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:36.068161 7640 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:43:36.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:36.068446 7640 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:43:36.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:36.068830 7640 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:43:36.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:36.069161 7640 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:43:36.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:36.069466 7640 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:43:36.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:36.069772 7640 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:43:36.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:36.070000 7640 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42693/jwks.json
06:43:36.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:36.106516 7640 master_runner.cc:386] Master server non-default flags:
06:43:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:36.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.46.148:42101
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/wal
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.46.190:32891,127.7.46.189:45741,127.7.46.188:43659
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.188.keytab
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.46.188
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.46.188:43659
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42693/jwks.json
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data/info.pb
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.46.188
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40401
06:43:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/logs
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:36.107973 7640 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:43:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:36.109356 7640 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:43:37.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:36.126262 7649 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:37.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:36.126271 7651 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:37.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:36.127753 7640 server_base.cc:1034] running on GCE node
06:43:37.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:36.126217 7648 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:37.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.360790 7640 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:43:37.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.363669 7640 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:43:37.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.365108 7640 hybrid_clock.cc:648] HybridClock initialized: now 1790664217365042 us; error 70 us; skew 500 ppm
06:43:37.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.365983 7640 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:43:37.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:37 dist-test-slave-vcpq krb5kdc[7366](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664217, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:43:37.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.371800 7640 init.cc:377] Logged in from keytab as kudu/127.7.46.188@KRBTEST.COM (short username kudu)
06:43:37.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.376368 7640 webserver.cc:460] Webserver started at http://127.7.46.188:40401/ using document root <none> and password file <none>
06:43:37.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.377342 7640 fs_manager.cc:359] Metadata directory not provided
06:43:37.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.377532 7640 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:43:37.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.385592 7640 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.002s sys 0.004s
06:43:37.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.389775 7660 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:43:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.390923 7640 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:43:37.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.391268 7640 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data,/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/wal
06:43:37.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9c2ec393d5534ef0a3398545ee67cdb8"
06:43:37.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:35 on dist-test-slave-vcpq"
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.393805 7640 fs_report.cc:383] FS layout report
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/wal
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/wal
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data/data
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:43:37.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:43:37.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:43:37.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.433328 7640 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:43:37.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.439368 7640 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:43:37.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.439884 7640 kserver.cc:163] Server-wide thread pool size limit: 3276
06:43:37.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.519984 7640 rpc_server.cc:307] RPC server started. Bound to: 127.7.46.188:43659
06:43:37.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.520174 7711 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.46.188:43659 every 8 connection(s)
06:43:37.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.522710 7640 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-2/data/info.pb
06:43:37.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.531512 7354 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu as pid 7640
06:43:37.532 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:43:37.532 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.46.189:45741
06:43:37.532 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
06:43:37.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.533289 7712 sys_catalog.cc:263] Verifying existing consensus state
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.534132 7354 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/logs
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data/info.pb
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:43:37.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.46.189:45741
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.46.189
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44729
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.189.keytab
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.46.189
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:43:37.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:43:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:43:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.46.148:42101
06:43:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.46.190:32891,127.7.46.189:45741,127.7.46.188:43659
06:43:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42693/jwks.json
06:43:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:43:37.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.541697 7712 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: Bootstrap starting.
06:43:37.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.574925 7712 log.cc:824] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: Log is configured to *not* fsync() on all Append() calls
06:43:37.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.580148 7712 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:43:37.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.580900 7712 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: Bootstrap complete.
06:43:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.606112 7712 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd9fd47cf55245a4bfe60cac955c07c1" member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } } peers { permanent_uuid: "b9fc30b95b2748bcab06730f45e93eae" member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } } peers { permanent_uuid: "9c2ec393d5534ef0a3398545ee67cdb8" member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } }
06:43:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.608976 7712 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:43:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.609277 7712 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9c2ec393d5534ef0a3398545ee67cdb8, State: Initialized, Role: FOLLOWER
06:43:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.610284 7712 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd9fd47cf55245a4bfe60cac955c07c1" member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } } peers { permanent_uuid: "b9fc30b95b2748bcab06730f45e93eae" member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } } peers { permanent_uuid: "9c2ec393d5534ef0a3398545ee67cdb8" member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } }
06:43:37.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.613970 7716 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd9fd47cf55245a4bfe60cac955c07c1" member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } } peers { permanent_uuid: "b9fc30b95b2748bcab06730f45e93eae" member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } } peers { permanent_uuid: "9c2ec393d5534ef0a3398545ee67cdb8" member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } } }
06:43:37.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.615054 7716 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [sys.catalog]: This master's current role is: FOLLOWER
06:43:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.615808 7712 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [sys.catalog]: configured and running, proceeding with master startup.
06:43:37.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.649997 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:37.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.650696 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:37.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.660012 7640 master_runner.cc:186] Error getting master registration for 127.7.46.190:32891: Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111),
06:43:37.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.690769 7640 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:43:37.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.694900 7666 tablet.cc:2369] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:43:37.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.756852 7716 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:43:37.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.757457 7716 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd9fd47cf55245a4bfe60cac955c07c1" member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } } peers { permanent_uuid: "b9fc30b95b2748bcab06730f45e93eae" member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } } peers { permanent_uuid: "9c2ec393d5534ef0a3398545ee67cdb8" member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } }
06:43:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.763756 7663 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111)
06:43:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.770643 7716 leader_election.cc:288] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bd9fd47cf55245a4bfe60cac955c07c1 (127.7.46.190:32891), b9fc30b95b2748bcab06730f45e93eae (127.7.46.189:45741)
06:43:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.780519 7663 leader_election.cc:334] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bd9fd47cf55245a4bfe60cac955c07c1 (127.7.46.190:32891): Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111)
06:43:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.785310 7661 leader_election.cc:334] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b9fc30b95b2748bcab06730f45e93eae (127.7.46.189:45741): Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:37.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.785857 7661 leader_election.cc:302] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9c2ec393d5534ef0a3398545ee67cdb8; no voters: b9fc30b95b2748bcab06730f45e93eae, bd9fd47cf55245a4bfe60cac955c07c1
06:43:37.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.786872 7716 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:43:37.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.918068 7714 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:43:37.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.918620 7714 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:43:37.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.919351 7714 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:43:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.950911 7714 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:43:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.951195 7714 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:43:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.951576 7714 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:43:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.951906 7714 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:43:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.952260 7714 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:43:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.952593 7714 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:43:37.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:37.952816 7714 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42693/jwks.json
06:43:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.993023 7714 master_runner.cc:386] Master server non-default flags:
06:43:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.46.148:42101
06:43:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data
06:43:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal
06:43:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:43:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.46.190:32891,127.7.46.189:45741,127.7.46.188:43659
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.189.keytab
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.46.189
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.46.189:45741
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42693/jwks.json
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data/info.pb
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.46.189
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44729
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/logs
06:43:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:43:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:43:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:43:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:43:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:43:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:43:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:43:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:43:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:43:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.994619 7714 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:43:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:37.995947 7714 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:43:38.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:38.015007 7739 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:38.655153 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:38.655488 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:39.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:38.018261 7714 server_base.cc:1034] running on GCE node
06:43:39.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:38.016292 7741 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:39.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:38.016495 7738 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:39.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.322151 7714 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:43:39.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.325372 7714 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:43:39.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.326954 7714 hybrid_clock.cc:648] HybridClock initialized: now 1790664219326924 us; error 55 us; skew 500 ppm
06:43:39.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.327914 7714 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:43:39.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:39 dist-test-slave-vcpq krb5kdc[7366](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664219, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:43:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.342357 7714 init.cc:377] Logged in from keytab as kudu/127.7.46.189@KRBTEST.COM (short username kudu)
06:43:39.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.348332 7714 webserver.cc:460] Webserver started at http://127.7.46.189:44729/ using document root <none> and password file <none>
06:43:39.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.349233 7714 fs_manager.cc:359] Metadata directory not provided
06:43:39.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.349465 7714 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:43:39.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.357999 7714 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
06:43:39.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.363124 7750 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:43:39.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.364373 7714 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.364722 7714 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data,/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9fc30b95b2748bcab06730f45e93eae"
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:33 on dist-test-slave-vcpq"
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.367046 7714 fs_report.cc:383] FS layout report
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data/data
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:43:39.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:43:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:43:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:43:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:43:39.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:43:39.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.428573 7714 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:43:39.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.436029 7714 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:43:39.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.436611 7714 kserver.cc:163] Server-wide thread pool size limit: 3276
06:43:39.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.534111 7714 rpc_server.cc:307] RPC server started. Bound to: 127.7.46.189:45741
06:43:39.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.534286 7801 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.46.189:45741 every 8 connection(s)
06:43:39.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.537825 7714 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/data/info.pb
06:43:39.542 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:43:39.542 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.7.46.190:32891
06:43:39.543 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
06:43:39.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.541051 7354 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu as pid 7714
06:43:39.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.545753 7354 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:39.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu
06:43:39.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal
06:43:39.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/logs
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data/info.pb
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.46.190:32891
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.46.190
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44603
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.190.keytab
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.46.190
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:43:39.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.46.148:42101
06:43:39.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:39.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:39.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:39.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:39.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.46.190:32891,127.7.46.189:45741,127.7.46.188:43659
06:43:39.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:39.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42693/jwks.json
06:43:39.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:43:39.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.551901 7802 sys_catalog.cc:263] Verifying existing consensus state
06:43:39.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 06:43:39.552917 7802 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:43:39.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.554008 7714 master.cc:537] Master@127.7.46.189:45741 shutting down...
06:43:39.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.624462 7714 master.cc:552] Master@127.7.46.189:45741 shutdown complete.
06:43:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:43:39.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.658389 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:39.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.658631 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:39.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.773599 7805 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:43:39.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.774358 7805 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd9fd47cf55245a4bfe60cac955c07c1" member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } } peers { permanent_uuid: "b9fc30b95b2748bcab06730f45e93eae" member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } } peers { permanent_uuid: "9c2ec393d5534ef0a3398545ee67cdb8" member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } }
06:43:39.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.776762 7805 leader_election.cc:288] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bd9fd47cf55245a4bfe60cac955c07c1 (127.7.46.190:32891), b9fc30b95b2748bcab06730f45e93eae (127.7.46.189:45741)
06:43:39.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.783224 7661 leader_election.cc:334] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b9fc30b95b2748bcab06730f45e93eae (127.7.46.189:45741): Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:39.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.783813 7663 leader_election.cc:334] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bd9fd47cf55245a4bfe60cac955c07c1 (127.7.46.190:32891): Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111)
06:43:39.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.784224 7663 leader_election.cc:302] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9c2ec393d5534ef0a3398545ee67cdb8; no voters: b9fc30b95b2748bcab06730f45e93eae, bd9fd47cf55245a4bfe60cac955c07c1
06:43:39.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.784888 7805 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:43:39.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.891634 7804 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:43:39.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.892160 7804 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:43:39.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.892879 7804 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:43:39.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.924360 7804 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:43:39.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.924630 7804 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:43:39.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.924901 7804 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:43:39.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.925096 7804 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:43:39.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.925287 7804 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:43:39.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.925467 7804 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:43:39.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.925603 7804 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42693/jwks.json
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.961634 7804 master_runner.cc:386] Master server non-default flags:
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.7.46.148:42101
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.7.46.190:32891,127.7.46.189:45741,127.7.46.188:43659
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/krb5kdc/kudu_127.7.46.190.keytab
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.7.46.190
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.7.46.190:32891
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42693/jwks.json
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data/info.pb
06:43:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.7.46.190
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44603
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/logs
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:43:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:43:39.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:43:39.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:43:39.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.962725 7804 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:43:39.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.963946 7804 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:43:39.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.982000 7815 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:40.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:40.661806 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:40.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:40.662079 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:41.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:39.983837 7804 server_base.cc:1034] running on GCE node
06:43:41.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.983247 7817 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:41.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:39.982033 7814 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:43:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.222669 7804 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:43:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.225595 7804 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:43:41.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.226976 7804 hybrid_clock.cc:648] HybridClock initialized: now 1790664221226942 us; error 40 us; skew 500 ppm
06:43:41.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.227821 7804 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:43:41.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:41 dist-test-slave-vcpq krb5kdc[7366](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664221, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:43:41.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.233924 7804 init.cc:377] Logged in from keytab as kudu/127.7.46.190@KRBTEST.COM (short username kudu)
06:43:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.240701 7804 webserver.cc:460] Webserver started at http://127.7.46.190:44603/ using document root <none> and password file <none>
06:43:41.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.241667 7804 fs_manager.cc:359] Metadata directory not provided
06:43:41.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.241925 7804 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:43:41.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.250108 7804 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.002s sys 0.004s
06:43:41.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.255062 7829 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:43:41.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.256215 7804 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
06:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.256479 7804 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data,/tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal
06:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd9fd47cf55245a4bfe60cac955c07c1"
06:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:43:31 on dist-test-slave-vcpq"
06:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.259055 7804 fs_report.cc:383] FS layout report
06:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal
06:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal
06:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data/data
06:43:41.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:43:41.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:43:41.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:43:41.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:43:41.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:43:41.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:43:41.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:43:41.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:43:41.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:43:41.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:43:41.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:43:41.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.312052 7804 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:43:41.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.317842 7804 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:43:41.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.318235 7804 kserver.cc:163] Server-wide thread pool size limit: 3276
06:43:41.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.387162 7804 rpc_server.cc:307] RPC server started. Bound to: 127.7.46.190:32891
06:43:41.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.387306 7880 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.7.46.190:32891 every 8 connection(s)
06:43:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.389760 7804 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/data/info.pb
06:43:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.394065 7354 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu as pid 7804
06:43:41.394 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:43:41.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.400750 7881 sys_catalog.cc:263] Verifying existing consensus state
06:43:41.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 06:43:41.401625 7881 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:43:41.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.402042 7804 master.cc:537] Master@127.7.46.190:32891 shutting down...
06:43:41.448 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.46.188:43659(127.7.46.188:43659)] connecting to peer
06:43:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.483250 7804 master.cc:552] Master@127.7.46.190:32891 shutdown complete.
06:43:41.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskUfq9I_/test-tmp/mini-kudu-cluster1755434709693899189/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:43:41.513 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:41.515 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:41.569 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.7.46.188:43659(127.7.46.188:43659)] Successfully connected to peer
06:43:41.572 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.573 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.574 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.633 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.633 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.634 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.653 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
06:43:41.661 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.7.46.188
06:43:41.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:41.666260 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:41.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:41.666594 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:41 dist-test-slave-vcpq krb5kdc[7366](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664215, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.7.46.188@KRBTEST.COM
06:43:41.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.880564 7887 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:43:41.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.880957 7887 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd9fd47cf55245a4bfe60cac955c07c1" member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } } peers { permanent_uuid: "b9fc30b95b2748bcab06730f45e93eae" member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } } peers { permanent_uuid: "9c2ec393d5534ef0a3398545ee67cdb8" member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } }
06:43:41.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.882301 7887 leader_election.cc:288] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bd9fd47cf55245a4bfe60cac955c07c1 (127.7.46.190:32891), b9fc30b95b2748bcab06730f45e93eae (127.7.46.189:45741)
06:43:41.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:41.890707 7661 leader_election.cc:334] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b9fc30b95b2748bcab06730f45e93eae (127.7.46.189:45741): Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:41.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:41.891330 7663 leader_election.cc:334] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bd9fd47cf55245a4bfe60cac955c07c1 (127.7.46.190:32891): Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111)
06:43:41.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.891742 7663 leader_election.cc:302] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9c2ec393d5534ef0a3398545ee67cdb8; no voters: b9fc30b95b2748bcab06730f45e93eae, bd9fd47cf55245a4bfe60cac955c07c1
06:43:41.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:41.892510 7887 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:43:41.897 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4cf88722, L:/127.0.0.1:34370 - R:/127.7.46.188:43659] using SASL/GSSAPI
06:43:41.913 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:41.922 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:41.924 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:41.927 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.927 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.928 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.931 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.931 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.932 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:41.939 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:41.941 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.941 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:41.941 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.942 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.945 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.946 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.947 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:41.959 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:41.961 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:41.962 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.962 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.964 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.966 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:41.980 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:41.981 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:41.983 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.983 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.984 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:41.985 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.985 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.986 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:41.987 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:42.000 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:42.001 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:42.003 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.003 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.005 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.007 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.007 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.007 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.008 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:42.079 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:42.081 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:42.082 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.082 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.083 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.083 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.083 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.084 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.085 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:42.180 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:42.181 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:42.183 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.183 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.184 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.186 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.186 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.187 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.188 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:42.399 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:42.400 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:42.401 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.402 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.403 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.404 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.404 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.404 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.405 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:42.520 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:42.521 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:42.525 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.527 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.527 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.527 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.529 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:42.669440 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:42.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:42.669737 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:42.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:42.694844 7640 master_runner.cc:186] Error getting master registration for 127.7.46.190:32891: Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111),
06:43:42.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:42.695329 7640 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:43:42.780 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:42.783 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.783 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.783 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:42.783 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:42.785 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.785 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:42.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:43.240 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:43.241 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:43.242 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:43.243 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:43.243 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:43.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:43.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:43.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:43.245 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:43.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:43.672583 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:43.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:43.672892 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:44.539 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:44.541 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:44.541 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:44.542 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:44.543 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:44.544 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:44.544 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:44.544 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:44.544 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:44.675789 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:44.676092 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:44.870291 7895 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:43:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:44.870723 7895 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd9fd47cf55245a4bfe60cac955c07c1" member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } } peers { permanent_uuid: "b9fc30b95b2748bcab06730f45e93eae" member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } } peers { permanent_uuid: "9c2ec393d5534ef0a3398545ee67cdb8" member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } }
06:43:44.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:44.872359 7895 leader_election.cc:288] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bd9fd47cf55245a4bfe60cac955c07c1 (127.7.46.190:32891), b9fc30b95b2748bcab06730f45e93eae (127.7.46.189:45741)
06:43:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:44.874248 7663 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111) [suppressed 5 similar messages]
06:43:44.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:44.876880 7661 leader_election.cc:334] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b9fc30b95b2748bcab06730f45e93eae (127.7.46.189:45741): Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:44.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:44.877303 7663 leader_election.cc:334] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bd9fd47cf55245a4bfe60cac955c07c1 (127.7.46.190:32891): Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111)
06:43:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:44.877652 7663 leader_election.cc:302] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9c2ec393d5534ef0a3398545ee67cdb8; no voters: b9fc30b95b2748bcab06730f45e93eae, bd9fd47cf55245a4bfe60cac955c07c1
06:43:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:44.878264 7895 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:43:45.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:45.679121 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:45.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:45.679425 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:46.683004 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:46.683347 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:47.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:47 dist-test-slave-vcpq krb5kdc[7366](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664227, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:43:47.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:47.375442 7656 init.cc:295] Successfully reacquired a new kerberos TGT
06:43:47.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:47.441881 6706 debug-util.cc:398] Leaking SignalData structure 0x7b08000b40e0 after lost signal to thread 6643
06:43:47.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:47.442515 6706 debug-util.cc:398] Leaking SignalData structure 0x7b0800040320 after lost signal to thread 6709
06:43:47.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:47.686401 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:47.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:47.686648 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:47.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:47.699344 7640 master_runner.cc:186] Error getting master registration for 127.7.46.190:32891: Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111),
06:43:47.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:47.699736 7640 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:43:48.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:48.203852 6781 debug-util.cc:398] Leaking SignalData structure 0x7b08000a35c0 after lost signal to thread 6716
06:43:48.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:48.204660 6781 debug-util.cc:398] Leaking SignalData structure 0x7b08000a9a80 after lost signal to thread 6784
06:43:48.361 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:48.362 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:48.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:48.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:48.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:48.366 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:48.366 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:48.366 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:48.367 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:48.690990 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:48.691803 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:49.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:49.253556 7906 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:43:49.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:49.254137 7906 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd9fd47cf55245a4bfe60cac955c07c1" member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } } peers { permanent_uuid: "b9fc30b95b2748bcab06730f45e93eae" member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } } peers { permanent_uuid: "9c2ec393d5534ef0a3398545ee67cdb8" member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } }
06:43:49.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:49.265295 7906 leader_election.cc:288] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bd9fd47cf55245a4bfe60cac955c07c1 (127.7.46.190:32891), b9fc30b95b2748bcab06730f45e93eae (127.7.46.189:45741)
06:43:49.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:49.274407 7663 leader_election.cc:334] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bd9fd47cf55245a4bfe60cac955c07c1 (127.7.46.190:32891): Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111)
06:43:49.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:49.275279 7661 leader_election.cc:334] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b9fc30b95b2748bcab06730f45e93eae (127.7.46.189:45741): Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:49.275715 7661 leader_election.cc:302] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9c2ec393d5534ef0a3398545ee67cdb8; no voters: b9fc30b95b2748bcab06730f45e93eae, bd9fd47cf55245a4bfe60cac955c07c1
06:43:49.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:49.276468 7906 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:43:49.479 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:49.480 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:49.483 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:49.483 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:49.483 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:49.483 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:49.483 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:49.483 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:49.485 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:49.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:49.695269 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:49.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:49.695534 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:50.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:50.698354 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:50.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:50.698602 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:51.620 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:51.621 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:51.622 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:51.622 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:51.622 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:51.624 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:51.625 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:51.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:51.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:51.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:51.701256 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:51.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:51.701498 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:52.702323 7640 master_runner.cc:186] Error getting master registration for 127.7.46.190:32891: Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111),
06:43:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:52.702803 7640 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:43:52.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:52.704649 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:52.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:52.704902 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:53.559 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:53.562 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:53.562 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:53.562 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:53.562 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:53.564 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:53.564 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:53.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:53.565 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:53.707902 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:53.708248 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:54.620 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:54.621 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:54.622 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:54.622 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:54.622 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:54.624 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:54.624 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:54.624 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:54.624 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:54.711014 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:54.711269 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:55.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:55.714586 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:55.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:55.714893 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:55.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:55.726287 7915 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:43:55.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:55.726656 7915 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bd9fd47cf55245a4bfe60cac955c07c1" member_type: VOTER last_known_addr { host: "127.7.46.190" port: 32891 } } peers { permanent_uuid: "b9fc30b95b2748bcab06730f45e93eae" member_type: VOTER last_known_addr { host: "127.7.46.189" port: 45741 } } peers { permanent_uuid: "9c2ec393d5534ef0a3398545ee67cdb8" member_type: VOTER last_known_addr { host: "127.7.46.188" port: 43659 } }
06:43:55.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:55.728430 7915 leader_election.cc:288] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bd9fd47cf55245a4bfe60cac955c07c1 (127.7.46.190:32891), b9fc30b95b2748bcab06730f45e93eae (127.7.46.189:45741)
06:43:55.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:55.730834 7663 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111) [suppressed 3 similar messages]
06:43:55.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:55.734072 7663 leader_election.cc:334] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bd9fd47cf55245a4bfe60cac955c07c1 (127.7.46.190:32891): Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111)
06:43:55.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:55.734450 7661 leader_election.cc:334] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b9fc30b95b2748bcab06730f45e93eae (127.7.46.189:45741): Network error: Client connection negotiation failed: client connection to 127.7.46.189:45741: connect: Connection refused (error 111)
06:43:55.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:55.734867 7661 leader_election.cc:302] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9c2ec393d5534ef0a3398545ee67cdb8; no voters: b9fc30b95b2748bcab06730f45e93eae, bd9fd47cf55245a4bfe60cac955c07c1
06:43:55.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:55.735597 7915 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:43:56.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:56.718715 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:56.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:56.719087 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:43:57 dist-test-slave-vcpq krb5kdc[7366](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664237, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:43:57.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:57.377964 7656 init.cc:295] Successfully reacquired a new kerberos TGT
06:43:57.560 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:57.561 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:57.562 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:57.563 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:57.563 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:57.564 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:57.564 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:57.564 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:57.565 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:57.705785 7640 master_runner.cc:186] Error getting master registration for 127.7.46.190:32891: Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111),
06:43:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:43:57.706176 7640 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:43:57.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:57.721783 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:57.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:57.722043 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:58.459 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:43:58.460 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:43:58.463 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:58.463 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:58.463 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:43:58.463 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:58.464 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:58.464 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:43:58.464 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:43:58.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:58.725939 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:58.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:58.726655 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:43:59.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:59.729723 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:43:59.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:43:59.729991 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:44:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:00.732838 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:44:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:00.733150 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:44:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:01.735895 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:44:01.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:01.736184 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:44:01.759 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:44:01.760 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:44:01.761 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:01.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:01.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:01.763 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:01.763 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:01.763 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:01.763 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:44:02.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:44:02.709160 7640 master_runner.cc:186] Error getting master registration for 127.7.46.190:32891: Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111),
06:44:02.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:44:02.709535 7640 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:44:02.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:02.739087 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:44:02.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:02.739352 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:44:03.720 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:44:03.721 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:44:03.721 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:03.722 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:03.722 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:03.722 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:03.722 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:03.722 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:03.724 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:44:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:03.741966 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:44:03.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:03.742223 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:44:04.539 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:44:04.540 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:44:04.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:04.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:04.541 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:04.541 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:04.541 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:04.541 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:04.542 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:44:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:04.744993 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:44:04.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:04.745244 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:44:05.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:05.747946 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:44:05.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:05.748188 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:44:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:06.143211 6860 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9860 after lost signal to thread 6792
06:44:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:06.143782 6860 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7ba0 after lost signal to thread 6863
06:44:06.380 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:44:06.380 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:44:06.381 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:06.382 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:06.382 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:06.382 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:06.382 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:06.382 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:06.386 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:44:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:06.752991 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:44:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:06.753435 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:44:07.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:44:07 dist-test-slave-vcpq krb5kdc[7366](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790664247, etypes {rep=17 tkt=17 ses=17}, kudu/127.7.46.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:44:07.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:44:07.380728 7656 init.cc:295] Successfully reacquired a new kerberos TGT
06:44:07.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:44:07.713263 7640 master_runner.cc:186] Error getting master registration for 127.7.46.190:32891: Network error: Client connection negotiation failed: client connection to 127.7.46.190:32891: connect: Connection refused (error 111),
06:44:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:44:07.713848 7640 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:44:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:07.757683 7727 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9c2ec393d5534ef0a3398545ee67cdb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:44:07.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:07.758015 7727 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:44:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:08.327415 7337 debug-util.cc:398] Leaking SignalData structure 0x7b08000b69e0 after lost signal to thread 7212
06:44:08.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:44:08.328181 7337 debug-util.cc:398] Leaking SignalData structure 0x7b08000af3c0 after lost signal to thread 7340
06:44:08.400 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.189:45741(127.7.46.189:45741)] connecting to peer
06:44:08.401 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.7.46.190:32891(127.7.46.190:32891)] connecting to peer
06:44:08.401 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:08.401 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.7.46.189:45741(127.7.46.189:45741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:08.401 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.189:45741: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741
06:44:08.401 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:08.401 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.190:32891(127.7.46.190:32891)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:08.402 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.7.46.190:32891: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
06:44:08.404 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891; will retry
06:44:08.410 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=27006), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [171ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [232ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [508ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [512ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [525ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [528ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [529ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [531ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [539ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [543ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [545ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [561ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [562ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [564ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [581ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [582ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [585ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [601ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [604ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [606ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [680ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [681ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [682ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [781ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [784ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [785ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [786ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [996ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1001ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1003ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1117ms] refreshing cache from master, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1128ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1377ms] refreshing cache from master, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1378ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1383ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1384ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1837ms] refreshing cache from master, [1837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1841ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1842ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1843ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1843ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [3136ms] refreshing cache from master, [3137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [3137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [3139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [3140ms] too many traces: truncated at 100 traces, deferred=Deferred@1727539644(state=PENDING, result=null, callback=(continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901), errback=(continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901)))
06:44:08.417 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.46.188:43659(127.7.46.188:43659)] disconnecting while in state READY
06:44:08.418 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.46.189:45741(127.7.46.189:45741)] disconnecting while in state TERMINATED
06:44:08.418 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.7.46.190:32891(127.7.46.190:32891)] disconnecting while in state TERMINATED
06:44:08.424 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:44:08.428 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:44:08.428 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.7.46.188:43659(127.7.46.188:43659)] handling channelInactive
06:44:08.428 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.7.46.188:43659(127.7.46.188:43659)] cleaning up while in state READY due to: connection closed
06:44:08.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:44:08.439601 7354 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu with pid 7640
06:44:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T06:44:09Z chronyd exiting
06:44:10.764 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:44:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:44:10.766014 6630 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu with pid 6931
06:44:10.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:44:10.790925 6630 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu with pid 7070
06:44:10.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:44:10.821779 6630 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu with pid 7211
06:44:10.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:44:10.850992 6630 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu with pid 6642
06:44:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:44:10.884076 6630 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu with pid 6712
06:44:10.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:44:10.913617 6630 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskUfq9I_/build/tsan/bin/kudu with pid 6787
06:44:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T06:44:12Z chronyd exiting
06:44:13.094 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=27006), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [171ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [232ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [508ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [512ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [525ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [528ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [529ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [531ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [539ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [543ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [545ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [561ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [562ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [564ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [581ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [582ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [585ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [601ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [604ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [606ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [680ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [681ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [682ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [781ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [784ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [785ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [786ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [996ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1001ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1003ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1117ms] refreshing cache from master, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1128ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1377ms] refreshing cache from master, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1378ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1383ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1384ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1837ms] refreshing cache from master, [1837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1841ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1842ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1843ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1843ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [3136ms] refreshing cache from master, [3137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [3137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [3139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [3140ms] too many traces: truncated at 100 traces, deferred=Deferred@1727539644(state=PENDING, result=null, callback=(continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901), errback=(continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.46.190:32891
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=27006), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [171ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [232ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [508ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [512ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [525ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [528ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [529ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [531ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [539ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [543ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [545ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [561ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [562ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [564ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [581ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [582ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [585ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [601ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [604ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [606ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [680ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [681ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [682ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [781ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [784ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [785ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [786ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [996ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1001ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1003ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1117ms] refreshing cache from master, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1128ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1377ms] refreshing cache from master, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1378ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1383ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1384ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1837ms] refreshing cache from master, [1837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1841ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1842ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1843ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1843ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [3136ms] refreshing cache from master, [3137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [3137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [3139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [3140ms] too many traces: truncated at 100 traces, deferred=Deferred@1727539644(state=PENDING, result=null, callback=(continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901), errback=(continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.46.190:32891
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
06:44:14.083 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
06:44:14.084 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=27006), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [171ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [232ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [508ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [512ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [525ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [528ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [529ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [531ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [539ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [543ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [545ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [561ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [562ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [564ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [581ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [582ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [585ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [601ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [604ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [606ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [680ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [681ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [682ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [781ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [784ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [785ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [786ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [996ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1001ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1003ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1117ms] refreshing cache from master, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1128ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1377ms] refreshing cache from master, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1378ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1383ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1384ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1837ms] refreshing cache from master, [1837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1841ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1842ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1843ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1843ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [3136ms] refreshing cache from master, [3137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [3137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [3139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [3140ms] too many traces: truncated at 100 traces, deferred=Deferred@1727539644(state=PENDING, result=null, callback=(continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901), errback=(continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.46.190:32891
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=27006), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [171ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [232ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [508ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [512ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [525ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [528ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [529ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [531ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [539ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [543ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [543ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [545ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [557ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [561ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [562ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [564ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [581ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [582ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [585ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [601ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [604ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [606ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [680ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [681ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [682ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [781ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [784ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [785ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [786ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [996ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [998ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1001ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1002ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1003ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1117ms] refreshing cache from master, [1117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1124ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1128ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1377ms] refreshing cache from master, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1378ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1381ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1383ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1384ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1837ms] refreshing cache from master, [1837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [1838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [1839ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [1841ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.189:45741: Network error: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741, [1842ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.188:43659: OK, [1843ms] Sub RPC ConnectToMaster: received response from server master-127.7.46.190:32891: Network error: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [1843ms] delaying RPC due to: Service unavailable: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891, [3136ms] refreshing cache from master, [3137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.188:43659, [3137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.189:45741, [3139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.7.46.190:32891, [3140ms] too many traces: truncated at 100 traces, deferred=Deferred@1727539644(state=PENDING, result=null, callback=(continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901), errback=(continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.7.46.188:43659,127.7.46.189:45741,127.7.46.190:32891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.189:45741(127.7.46.189:45741): Connection refused: /127.7.46.189:45741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.7.46.190:32891(127.7.46.190:32891): Connection refused: /127.7.46.190:32891
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.7.46.190:32891
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)