15:09:59.629 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:09:59.640 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskY4X0Fv/test-tmp
15:09:59.643 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
15:09:59.646 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskY4X0Fv/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
15:09:59.861 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:10:00.013 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:10:00.015 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:10:00.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.017666 16877 mini_oidc.cc:152] Starting JWKS server
15:10:00.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.020845 16877 webserver.cc:460] Webserver started at http://127.0.0.1:34803/ using document root <none> and password file <none>
15:10:00.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.021142 16877 mini_oidc.cc:189] Starting OIDC Discovery server
15:10:00.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.021906 16877 webserver.cc:460] Webserver started at http://127.0.0.1:33199/ using document root <none> and password file <none>
15:10:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T15:10:00Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
15:10:00.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T15:10:00Z Disabled control of system clock
15:10:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.047147 16877 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/wal
15:10:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/data
15:10:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:00.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/logs
15:10:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/data/info.pb
15:10:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:00.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.123.126:46353
15:10:00.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.123.126
15:10:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.123.84:35073
15:10:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.123.126:46353,127.16.123.125:33429,127.16.123.124:35659
15:10:00.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:00.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34803/jwks.json with env {}
15:10:00.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.344549 16888 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:00.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.344820 16888 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:00.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.344902 16888 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:00.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.355660 16888 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:00.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.355793 16888 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:00.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.355851 16888 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:00.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.355917 16888 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:00.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.355967 16888 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:00.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.356029 16888 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34803/jwks.json
15:10:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.368829 16888 master_runner.cc:386] Master server non-default flags:
15:10:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.123.84:35073
15:10:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:00.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:00.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/data
15:10:00.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/wal
15:10:00.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:00.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.123.126:46353,127.16.123.125:33429,127.16.123.124:35659
15:10:00.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:00.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:00.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:00.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.123.126:46353
15:10:00.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:00.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:00.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:00.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34803/jwks.json
15:10:00.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:00.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:00.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/data/info.pb
15:10:00.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.123.126
15:10:00.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:00.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:00.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:00.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:00.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:00.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:00.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/logs
15:10:00.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:00.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:00.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:00.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:00.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:10:00.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:10:00.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:00.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 13:46:26 UTC on ccycloud.quasar-xkqhuq.root.comops.site
15:10:00.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676080
15:10:00.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:10:00.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.370205 16888 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:00.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.371850 16888 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:10:00.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.384511 16893 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:00.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.384485 16896 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:00.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.384570 16894 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:00.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.384797 16888 server_base.cc:1034] running on GCE node
15:10:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.386006 16888 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:00.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.387475 16888 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:00.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.388715 16888 hybrid_clock.cc:648] HybridClock initialized: now 1790521800388639 us; error 46 us; skew 500 ppm
15:10:00.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.389043 16888 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:00.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.410060 16888 webserver.cc:460] Webserver started at http://127.16.123.126:35445/ using document root <none> and password file <none>
15:10:00.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.410786 16888 fs_manager.cc:359] Metadata directory not provided
15:10:00.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.410914 16888 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:00.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.411175 16888 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:10:00.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.413798 16888 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/data/instance:
15:10:00.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f4ce2f0b9867465b838f8bbf68258563"
15:10:00.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:00 on dist-test-slave-kz7d"
15:10:00.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.414544 16888 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/wal/instance:
15:10:00.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f4ce2f0b9867465b838f8bbf68258563"
15:10:00.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:00 on dist-test-slave-kz7d"
15:10:00.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.419595 16888 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
15:10:00.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.422936 16902 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:00.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.424189 16888 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
15:10:00.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.424355 16888 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/data,/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/wal
15:10:00.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f4ce2f0b9867465b838f8bbf68258563"
15:10:00.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:00 on dist-test-slave-kz7d"
15:10:00.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.424571 16888 fs_report.cc:383] FS layout report
15:10:00.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:00.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/wal
15:10:00.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/wal
15:10:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/data/data
15:10:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:00.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.448094 16888 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:10:00.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.449051 16888 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:00.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.449433 16888 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:00.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.467094 16888 rpc_server.cc:307] RPC server started. Bound to: 127.16.123.126:46353
15:10:00.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.467133 16953 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.123.126:46353 every 8 connection(s)
15:10:00.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.468534 16888 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/data/info.pb
15:10:00.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.471596 16877 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu as pid 16888
15:10:00.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.471855 16877 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-0/wal/instance
15:10:00.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.472754 16954 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:10:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.474546 16877 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/wal
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/data
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/logs
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/data/info.pb
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:00.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.123.125:33429
15:10:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.123.125
15:10:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.123.84:35073
15:10:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.123.126:46353,127.16.123.125:33429,127.16.123.124:35659
15:10:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34803/jwks.json with env {}
15:10:00.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.481001 16954 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } has no permanent_uuid. Determining permanent_uuid...
15:10:00.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.497239 16954 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } has no permanent_uuid. Determining permanent_uuid...
15:10:00.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.498347 16906 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.123.125:33429: connect: Connection refused (error 111)
15:10:00.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.499949 16954 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.123.125:33429: Network error: Client connection negotiation failed: client connection to 127.16.123.125:33429: connect: Connection refused (error 111)
15:10:00.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.549326 16954 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } attempt: 1
15:10:00.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.552784 16954 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.123.125:33429: Network error: Client connection negotiation failed: client connection to 127.16.123.125:33429: connect: Connection refused (error 111)
15:10:00.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.621083 16954 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } attempt: 2
15:10:00.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.623694 16954 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.123.125:33429: Network error: Client connection negotiation failed: client connection to 127.16.123.125:33429: connect: Connection refused (error 111)
15:10:00.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.714959 16954 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } attempt: 3
15:10:00.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.717348 16954 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.123.125:33429: Network error: Client connection negotiation failed: client connection to 127.16.123.125:33429: connect: Connection refused (error 111)
15:10:00.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.784379 16956 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:00.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.784648 16956 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:00.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.784731 16956 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:00.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.791980 16956 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:00.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.792078 16956 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:00.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.792119 16956 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:00.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.792163 16956 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:00.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.792198 16956 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:00.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.792244 16956 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34803/jwks.json
15:10:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.801851 16956 master_runner.cc:386] Master server non-default flags:
15:10:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.123.84:35073
15:10:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/data
15:10:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/wal
15:10:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.123.126:46353,127.16.123.125:33429,127.16.123.124:35659
15:10:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.123.125:33429
15:10:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34803/jwks.json
15:10:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:00.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/data/info.pb
15:10:00.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.123.125
15:10:00.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:00.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:00.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:00.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:00.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:00.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:00.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/logs
15:10:00.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:00.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:00.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:00.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:00.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:10:00.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:10:00.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:00.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 13:46:26 UTC on ccycloud.quasar-xkqhuq.root.comops.site
15:10:00.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676080
15:10:00.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:10:00.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.803504 16956 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:00.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.805830 16956 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:10:00.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.816915 16965 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:00.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.816977 16964 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:00.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.817044 16967 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:00.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.817566 16956 server_base.cc:1034] running on GCE node
15:10:00.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.818192 16956 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:00.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.819394 16956 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.820626 16956 hybrid_clock.cc:648] HybridClock initialized: now 1790521800820554 us; error 46 us; skew 500 ppm
15:10:00.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.820909 16956 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.823452 16956 webserver.cc:460] Webserver started at http://127.16.123.125:46667/ using document root <none> and password file <none>
15:10:00.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.824198 16956 fs_manager.cc:359] Metadata directory not provided
15:10:00.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.824491 16956 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:00.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.824865 16956 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:10:00.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.827670 16956 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/data/instance:
15:10:00.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab34f94d47284c97bd400c48e4710b08"
15:10:00.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:00 on dist-test-slave-kz7d"
15:10:00.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.829015 16956 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/wal/instance:
15:10:00.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab34f94d47284c97bd400c48e4710b08"
15:10:00.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:00 on dist-test-slave-kz7d"
15:10:00.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.834972 16956 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
15:10:00.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.838358 16973 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.840098 16956 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
15:10:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.840255 16956 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/data,/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/wal
15:10:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab34f94d47284c97bd400c48e4710b08"
15:10:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:00 on dist-test-slave-kz7d"
15:10:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.840494 16956 fs_report.cc:383] FS layout report
15:10:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/wal
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/wal
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/data/data
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:00.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.853868 16956 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:10:00.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.854692 16956 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:00.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.855018 16956 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:00.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.860575 16954 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } attempt: 4
15:10:00.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.862550 16954 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.123.125:33429: Network error: Client connection negotiation failed: client connection to 127.16.123.125:33429: connect: Connection refused (error 111)
15:10:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.872521 16956 rpc_server.cc:307] RPC server started. Bound to: 127.16.123.125:33429
15:10:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.872539 17024 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.123.125:33429 every 8 connection(s)
15:10:00.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.873893 16956 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/data/info.pb
15:10:00.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.878157 17025 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:10:00.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.880563 16877 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu as pid 16956
15:10:00.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.880723 16877 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-1/wal/instance
15:10:00.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.881770 16877 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/wal
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/data
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/logs
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/data/info.pb
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:00.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.123.124:35659
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.123.124
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.123.84:35073
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.123.126:46353,127.16.123.125:33429,127.16.123.124:35659
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34803/jwks.json with env {}
15:10:00.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.884081 17025 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } has no permanent_uuid. Determining permanent_uuid...
15:10:00.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.898530 17025 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } has no permanent_uuid. Determining permanent_uuid...
15:10:00.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.907630 17025 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } has no permanent_uuid. Determining permanent_uuid...
15:10:00.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.908581 16976 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.123.124:35659: connect: Connection refused (error 111)
15:10:00.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.910028 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.123.124:35659: Network error: Client connection negotiation failed: client connection to 127.16.123.124:35659: connect: Connection refused (error 111)
15:10:00.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:00.959303 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } attempt: 1
15:10:00.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:00.961124 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.123.124:35659: Network error: Client connection negotiation failed: client connection to 127.16.123.124:35659: connect: Connection refused (error 111)
15:10:01.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.029377 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } attempt: 2
15:10:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.031472 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.123.124:35659: Network error: Client connection negotiation failed: client connection to 127.16.123.124:35659: connect: Connection refused (error 111)
15:10:01.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.122728 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } attempt: 3
15:10:01.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.125419 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.123.124:35659: Network error: Client connection negotiation failed: client connection to 127.16.123.124:35659: connect: Connection refused (error 111)
15:10:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.161819 16954 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } attempt: 5
15:10:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.170428 16954 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } has no permanent_uuid. Determining permanent_uuid...
15:10:01.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.172309 16954 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.123.124:35659: Network error: Client connection negotiation failed: client connection to 127.16.123.124:35659: connect: Connection refused (error 111)
15:10:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.195971 17027 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.196274 17027 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.196391 17027 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.203872 17027 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.203992 17027 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.204051 17027 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.204110 17027 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.204157 17027 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:01.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.204219 17027 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34803/jwks.json
15:10:01.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.214285 17027 master_runner.cc:386] Master server non-default flags:
15:10:01.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:01.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.123.84:35073
15:10:01.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:01.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/data
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/wal
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.123.126:46353,127.16.123.125:33429,127.16.123.124:35659
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.123.124:35659
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34803/jwks.json
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/data/info.pb
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.123.124
15:10:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/logs
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 13:46:26 UTC on ccycloud.quasar-xkqhuq.root.comops.site
15:10:01.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676080
15:10:01.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:10:01.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.215973 17027 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:01.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.218046 17027 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:10:01.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.223536 16954 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } attempt: 1
15:10:01.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.226869 16954 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.123.124:35659: Network error: Client connection negotiation failed: client connection to 127.16.123.124:35659: connect: Connection refused (error 111)
15:10:01.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.230966 17036 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:01.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.231163 17038 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:01.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.230914 17035 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:01.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.233798 17027 server_base.cc:1034] running on GCE node
15:10:01.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.234982 17027 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:01.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.236831 17027 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:01.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.238390 17027 hybrid_clock.cc:648] HybridClock initialized: now 1790521801238315 us; error 37 us; skew 500 ppm
15:10:01.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.238780 17027 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:01.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.241479 17027 webserver.cc:460] Webserver started at http://127.16.123.124:36469/ using document root <none> and password file <none>
15:10:01.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.242271 17027 fs_manager.cc:359] Metadata directory not provided
15:10:01.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.242393 17027 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:01.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.242707 17027 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:10:01.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.246265 17027 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/data/instance:
15:10:01.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ba101400efe4893b88b7fc1b574019d"
15:10:01.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:01 on dist-test-slave-kz7d"
15:10:01.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.247201 17027 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/wal/instance:
15:10:01.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ba101400efe4893b88b7fc1b574019d"
15:10:01.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:01 on dist-test-slave-kz7d"
15:10:01.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.253595 17027 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.003s
15:10:01.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.257761 17044 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.259275 17027 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
15:10:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.259445 17027 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/data,/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/wal
15:10:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ba101400efe4893b88b7fc1b574019d"
15:10:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:01 on dist-test-slave-kz7d"
15:10:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.259654 17027 fs_report.cc:383] FS layout report
15:10:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/wal
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/wal
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/data/data
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:01.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.268664 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } attempt: 4
15:10:01.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.270350 17025 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.123.124:35659: Network error: Client connection negotiation failed: client connection to 127.16.123.124:35659: connect: Connection refused (error 111)
15:10:01.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.281476 17027 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:10:01.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.282433 17027 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.282773 17027 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:01.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.295096 16954 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } attempt: 2
15:10:01.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.297639 16954 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.123.124:35659: Network error: Client connection negotiation failed: client connection to 127.16.123.124:35659: connect: Connection refused (error 111)
15:10:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.305598 17027 rpc_server.cc:307] RPC server started. Bound to: 127.16.123.124:35659
15:10:01.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.305616 17095 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.123.124:35659 every 8 connection(s)
15:10:01.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.306843 17027 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/data/info.pb
15:10:01.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.307205 16877 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu as pid 17027
15:10:01.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.307376 16877 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/master-2/wal/instance
15:10:01.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.311488 17096 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:10:01.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.319278 17096 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } has no permanent_uuid. Determining permanent_uuid...
15:10:01.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.333902 17096 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } has no permanent_uuid. Determining permanent_uuid...
15:10:01.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.340847 17096 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } has no permanent_uuid. Determining permanent_uuid...
15:10:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.351542 17096 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d: Bootstrap starting.
15:10:01.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.354475 17096 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d: Neither blocks nor log segments found. Creating new log.
15:10:01.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.355583 17096 log.cc:824] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d: Log is configured to *not* fsync() on all Append() calls
15:10:01.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.358007 17096 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d: No bootstrap required, opened a new log
15:10:01.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.365694 17096 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } }
15:10:01.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.365989 17096 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:10:01.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.366062 17096 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9ba101400efe4893b88b7fc1b574019d, State: Initialized, Role: FOLLOWER
15:10:01.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.366565 17096 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } }
15:10:01.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.367561 17103 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } } }
15:10:01.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.367967 17103 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [sys.catalog]: This master's current role is: FOLLOWER
15:10:01.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.368824 17096 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [sys.catalog]: configured and running, proceeding with master startup.
15:10:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.379971 17114 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.380105 17114 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:01.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.376529 17027 master_runner.cc:186] Error getting master registration for 127.16.123.126:46353: OK, instance_id { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" instance_seqno: 1790521800462309 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:10:01.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.385705 17027 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:01.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.403879 16954 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } attempt: 3
15:10:01.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.413496 16954 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563: Bootstrap starting.
15:10:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.417173 16954 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563: Neither blocks nor log segments found. Creating new log.
15:10:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.418162 16954 log.cc:824] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563: Log is configured to *not* fsync() on all Append() calls
15:10:01.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.420485 16954 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563: No bootstrap required, opened a new log
15:10:01.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.427695 16954 raft_consensus.cc:348] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } }
15:10:01.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.428053 16954 raft_consensus.cc:374] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:10:01.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.428165 16954 raft_consensus.cc:729] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f4ce2f0b9867465b838f8bbf68258563, State: Initialized, Role: FOLLOWER
15:10:01.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.429486 16954 consensus_queue.cc:260] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } }
15:10:01.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.430444 17117 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } } }
15:10:01.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.430810 17117 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [sys.catalog]: This master's current role is: FOLLOWER
15:10:01.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.431591 16954 sys_catalog.cc:564] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [sys.catalog]: configured and running, proceeding with master startup.
15:10:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.440912 17128 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.441396 17128 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:01.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.452332 16908 tablet.cc:2369] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:10:01.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.440551 16888 master_runner.cc:186] Error getting master registration for 127.16.123.125:33429: OK, instance_id { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" instance_seqno: 1790521800867691 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:10:01.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.452986 16888 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:01.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.535243 17050 tablet.cc:2369] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:10:01.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.569561 17025 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } attempt: 5
15:10:01.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.588636 17025 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08: Bootstrap starting.
15:10:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.593082 17025 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08: Neither blocks nor log segments found. Creating new log.
15:10:01.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.594404 17025 log.cc:824] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08: Log is configured to *not* fsync() on all Append() calls
15:10:01.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.599359 17025 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08: No bootstrap required, opened a new log
15:10:01.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.610374 17025 raft_consensus.cc:348] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } }
15:10:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.610790 17025 raft_consensus.cc:374] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:10:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.610898 17025 raft_consensus.cc:729] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ab34f94d47284c97bd400c48e4710b08, State: Initialized, Role: FOLLOWER
15:10:01.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.611632 17025 consensus_queue.cc:260] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } }
15:10:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.612624 17131 sys_catalog.cc:455] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } } }
15:10:01.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.613238 17131 sys_catalog.cc:458] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [sys.catalog]: This master's current role is: FOLLOWER
15:10:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.614274 17025 sys_catalog.cc:564] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [sys.catalog]: configured and running, proceeding with master startup.
15:10:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.623330 17142 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:01.623718 17142 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:01.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.632586 16956 master_runner.cc:263] No leader master found from master ab34f94d47284c97bd400c48e4710b08
15:10:01.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.632730 16956 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:01.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.765010 17117 raft_consensus.cc:482] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:01.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.765347 17117 raft_consensus.cc:504] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } }
15:10:01.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.766960 17117 leader_election.cc:288] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ab34f94d47284c97bd400c48e4710b08 (127.16.123.125:33429), 9ba101400efe4893b88b7fc1b574019d (127.16.123.124:35659)
15:10:01.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.768079 17000 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f4ce2f0b9867465b838f8bbf68258563" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ab34f94d47284c97bd400c48e4710b08" is_pre_election: true
15:10:01.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.768664 17000 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f4ce2f0b9867465b838f8bbf68258563 in term 0.
15:10:01.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.769599 16906 leader_election.cc:302] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ab34f94d47284c97bd400c48e4710b08, f4ce2f0b9867465b838f8bbf68258563; no voters:
15:10:01.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.771483 17117 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:10:01.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.771653 17117 raft_consensus.cc:482] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:10:01.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.771737 17117 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [term 0 FOLLOWER]: Advancing to term 1
15:10:01.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.773309 17071 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f4ce2f0b9867465b838f8bbf68258563" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9ba101400efe4893b88b7fc1b574019d" is_pre_election: true
15:10:01.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.773823 17071 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f4ce2f0b9867465b838f8bbf68258563 in term 0.
15:10:01.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.776014 17117 raft_consensus.cc:504] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } }
15:10:01.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.777266 17071 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f4ce2f0b9867465b838f8bbf68258563" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9ba101400efe4893b88b7fc1b574019d"
15:10:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.777311 17000 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f4ce2f0b9867465b838f8bbf68258563" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ab34f94d47284c97bd400c48e4710b08"
15:10:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.777469 17071 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [term 0 FOLLOWER]: Advancing to term 1
15:10:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.777549 17000 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [term 0 FOLLOWER]: Advancing to term 1
15:10:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.780396 17071 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f4ce2f0b9867465b838f8bbf68258563 in term 1.
15:10:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.781251 17000 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f4ce2f0b9867465b838f8bbf68258563 in term 1.
15:10:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.783329 16906 leader_election.cc:302] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ab34f94d47284c97bd400c48e4710b08, f4ce2f0b9867465b838f8bbf68258563; no voters:
15:10:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.790813 17117 leader_election.cc:288] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [CANDIDATE]: Term 1 election: Requested vote from peers ab34f94d47284c97bd400c48e4710b08 (127.16.123.125:33429), 9ba101400efe4893b88b7fc1b574019d (127.16.123.124:35659)
15:10:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.791087 17117 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [term 1 FOLLOWER]: Leader election won for term 1
15:10:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.791226 17117 raft_consensus.cc:686] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [term 1 LEADER]: Becoming Leader. State: Replica: f4ce2f0b9867465b838f8bbf68258563, State: Running, Role: LEADER
15:10:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.791795 17117 consensus_queue.cc:237] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } }
15:10:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.799167 17144 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f4ce2f0b9867465b838f8bbf68258563. Latest consensus state: current_term: 1 leader_uuid: "f4ce2f0b9867465b838f8bbf68258563" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } } }
15:10:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.799883 17144 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [sys.catalog]: This master's current role is: LEADER
15:10:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.801388 17146 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
15:10:01.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.803639 17146 catalog_manager.cc:1484] Initializing Kudu cluster ID...
15:10:01.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.813117 17071 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [term 1 FOLLOWER]: Refusing update from remote peer f4ce2f0b9867465b838f8bbf68258563: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:10:01.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.813252 17000 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [term 1 FOLLOWER]: Refusing update from remote peer f4ce2f0b9867465b838f8bbf68258563: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:10:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.814234 17144 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:10:01.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.814697 17117 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:10:01.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.844213 17131 sys_catalog.cc:455] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f4ce2f0b9867465b838f8bbf68258563. Latest consensus state: current_term: 1 leader_uuid: "f4ce2f0b9867465b838f8bbf68258563" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } } }
15:10:01.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.844547 17131 sys_catalog.cc:458] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [sys.catalog]: This master's current role is: FOLLOWER
15:10:01.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.847220 17103 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [sys.catalog]: SysCatalogTable state changed. Reason: New leader f4ce2f0b9867465b838f8bbf68258563. Latest consensus state: current_term: 1 leader_uuid: "f4ce2f0b9867465b838f8bbf68258563" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } } }
15:10:01.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.847492 17103 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [sys.catalog]: This master's current role is: FOLLOWER
15:10:01.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.847601 17147 mvcc.cc:205] Tried to move back new op lower bound from 7333977300215832576 to 7333977300152111104. Current Snapshot: MvccSnapshot[applied={T|T < 7333977300215832576}]
15:10:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.857046 17131 sys_catalog.cc:455] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f4ce2f0b9867465b838f8bbf68258563" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } } }
15:10:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.857131 17144 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f4ce2f0b9867465b838f8bbf68258563" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } } }
15:10:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.857407 17131 sys_catalog.cc:458] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08 [sys.catalog]: This master's current role is: FOLLOWER
15:10:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.857414 17144 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [sys.catalog]: This master's current role is: LEADER
15:10:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.860741 17146 catalog_manager.cc:1347] Generated new cluster ID: f1bb51645d05440d9a0bd384e4f64d31
15:10:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.860850 17146 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
15:10:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.861163 17144 sys_catalog.cc:455] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f4ce2f0b9867465b838f8bbf68258563" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } } }
15:10:01.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.861477 17144 sys_catalog.cc:458] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563 [sys.catalog]: This master's current role is: LEADER
15:10:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.875484 17103 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f4ce2f0b9867465b838f8bbf68258563" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f4ce2f0b9867465b838f8bbf68258563" member_type: VOTER last_known_addr { host: "127.16.123.126" port: 46353 } } peers { permanent_uuid: "ab34f94d47284c97bd400c48e4710b08" member_type: VOTER last_known_addr { host: "127.16.123.125" port: 33429 } } peers { permanent_uuid: "9ba101400efe4893b88b7fc1b574019d" member_type: VOTER last_known_addr { host: "127.16.123.124" port: 35659 } } }
15:10:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.875775 17103 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d [sys.catalog]: This master's current role is: FOLLOWER
15:10:01.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.908392 17146 catalog_manager.cc:1370] Generated new certificate authority record
15:10:01.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.911259 17146 catalog_manager.cc:1504] Loading token signing keys...
15:10:01.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.933610 17146 catalog_manager.cc:5868] T 00000000000000000000000000000000 P f4ce2f0b9867465b838f8bbf68258563: Generated new TSK 0
15:10:01.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.935439 17146 catalog_manager.cc:1514] Initializing in-progress tserver states...
15:10:01.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:01.958693 16877 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:01.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/wal
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/data
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/logs
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/data/info.pb
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.123.65:0
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.123.65
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.123.65
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.123.126:46353,127.16.123.125:33429,127.16.123.124:35659
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.123.84:35073
15:10:01.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:01.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:01.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:10:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.290057 17156 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.290338 17156 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.290414 17156 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.297530 17156 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.297695 17156 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.123.65
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.306774 17156 tablet_server_runner.cc:78] Tablet server non-default flags:
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.123.84:35073
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/data
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/wal
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.123.65:0
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/data/info.pb
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.123.65
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.123.126:46353,127.16.123.125:33429,127.16.123.124:35659
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.123.65
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/logs
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 13:46:26 UTC on ccycloud.quasar-xkqhuq.root.comops.site
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676080
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:10:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.307974 17156 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:02.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.309764 17156 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:10:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.322022 17161 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.322018 17162 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:02.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.322423 17164 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.323601 17156 server_base.cc:1034] running on GCE node
15:10:02.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.324165 17156 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:02.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.325045 17156 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:02.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.326264 17156 hybrid_clock.cc:648] HybridClock initialized: now 1790521802326194 us; error 37 us; skew 500 ppm
15:10:02.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.326566 17156 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:02.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.329407 17156 webserver.cc:460] Webserver started at http://127.16.123.65:37151/ using document root <none> and password file <none>
15:10:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.330088 17156 fs_manager.cc:359] Metadata directory not provided
15:10:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.330207 17156 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.330531 17156 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:10:02.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.333249 17156 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/data/instance:
15:10:02.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9463a56be954610826bd7be223f6bf1"
15:10:02.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:02 on dist-test-slave-kz7d"
15:10:02.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.334048 17156 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/wal/instance:
15:10:02.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9463a56be954610826bd7be223f6bf1"
15:10:02.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:02 on dist-test-slave-kz7d"
15:10:02.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.338795 17156 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
15:10:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.342260 17170 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.343632 17156 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
15:10:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.343760 17156 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/data,/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/wal
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d9463a56be954610826bd7be223f6bf1"
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:02 on dist-test-slave-kz7d"
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.343933 17156 fs_report.cc:383] FS layout report
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/wal
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/wal
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/data/data
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:02.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.365918 17156 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:10:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.367056 17156 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.367363 17156 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:02.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.368315 17156 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:10:02.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.369988 17156 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:10:02.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.370060 17156 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:10:02.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.370167 17156 ts_tablet_manager.cc:610] Registered 0 tablets
15:10:02.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.370208 17156 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:10:02.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.382431 17114 catalog_manager.cc:1259] Loaded cluster ID: f1bb51645d05440d9a0bd384e4f64d31
15:10:02.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.382530 17114 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d: loading cluster ID for follower catalog manager: success
15:10:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.385986 17114 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d: acquiring CA information for follower catalog manager: success
15:10:02.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.388406 17114 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 9ba101400efe4893b88b7fc1b574019d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:10:02.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.402978 17156 rpc_server.cc:307] RPC server started. Bound to: 127.16.123.65:45861
15:10:02.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.403007 17282 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.123.65:45861 every 8 connection(s)
15:10:02.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.404161 17156 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/data/info.pb
15:10:02.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.404711 16877 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu as pid 17156
15:10:02.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.405078 16877 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-0/wal/instance
15:10:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.413295 16877 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/wal
15:10:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/data
15:10:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/logs
15:10:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/data/info.pb
15:10:02.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.123.66:0
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.123.66
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.123.66
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.123.126:46353,127.16.123.125:33429,127.16.123.124:35659
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.123.84:35073
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:02.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:10:02.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.436048 17283 heartbeater.cc:347] Connected to a master server at 127.16.123.124:35659
15:10:02.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.436232 17285 heartbeater.cc:347] Connected to a master server at 127.16.123.125:33429
15:10:02.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.436414 17285 heartbeater.cc:464] Registering TS with master...
15:10:02.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.436466 17284 heartbeater.cc:347] Connected to a master server at 127.16.123.126:46353
15:10:02.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.436569 17284 heartbeater.cc:464] Registering TS with master...
15:10:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.436414 17283 heartbeater.cc:464] Registering TS with master...
15:10:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.437503 17285 heartbeater.cc:511] Master 127.16.123.125:33429 requested a full tablet report, sending...
15:10:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.437633 17283 heartbeater.cc:511] Master 127.16.123.124:35659 requested a full tablet report, sending...
15:10:02.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.438097 17284 heartbeater.cc:511] Master 127.16.123.126:46353 requested a full tablet report, sending...
15:10:02.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.439560 16990 ts_manager.cc:194] Registered new tserver with Master: d9463a56be954610826bd7be223f6bf1 (127.16.123.65:45861)
15:10:02.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.439570 16919 ts_manager.cc:194] Registered new tserver with Master: d9463a56be954610826bd7be223f6bf1 (127.16.123.65:45861)
15:10:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.441478 16919 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.123.65:59981
15:10:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.442591 17061 ts_manager.cc:194] Registered new tserver with Master: d9463a56be954610826bd7be223f6bf1 (127.16.123.65:45861)
15:10:02.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.625802 17142 catalog_manager.cc:1259] Loaded cluster ID: f1bb51645d05440d9a0bd384e4f64d31
15:10:02.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.625896 17142 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08: loading cluster ID for follower catalog manager: success
15:10:02.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.628429 17142 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08: acquiring CA information for follower catalog manager: success
15:10:02.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.630245 17142 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ab34f94d47284c97bd400c48e4710b08: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:10:02.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.749275 17292 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:02.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.749706 17292 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:02.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.749806 17292 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:02.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.758386 17292 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:02.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.758582 17292 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.123.66
15:10:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.768209 17292 tablet_server_runner.cc:78] Tablet server non-default flags:
15:10:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.123.84:35073
15:10:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/data
15:10:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/wal
15:10:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.123.66:0
15:10:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/data/info.pb
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.123.66
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.123.126:46353,127.16.123.125:33429,127.16.123.124:35659
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.123.66
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/logs
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 13:46:26 UTC on ccycloud.quasar-xkqhuq.root.comops.site
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676080
15:10:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:10:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.770181 17292 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.772676 17292 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:10:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.787897 17299 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.787968 17301 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:02.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:02.788443 17298 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:02.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.793349 17292 server_base.cc:1034] running on GCE node
15:10:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.794215 17292 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.795585 17292 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.796881 17292 hybrid_clock.cc:648] HybridClock initialized: now 1790521802796812 us; error 74 us; skew 500 ppm
15:10:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.797179 17292 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.799804 17292 webserver.cc:460] Webserver started at http://127.16.123.66:44391/ using document root <none> and password file <none>
15:10:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.800448 17292 fs_manager.cc:359] Metadata directory not provided
15:10:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.800534 17292 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.800779 17292 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:10:02.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.803323 17292 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/data/instance:
15:10:02.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "799c4434881c4cb889a6fbbd7e6ae196"
15:10:02.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:02 on dist-test-slave-kz7d"
15:10:02.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.804126 17292 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/wal/instance:
15:10:02.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "799c4434881c4cb889a6fbbd7e6ae196"
15:10:02.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:02 on dist-test-slave-kz7d"
15:10:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.809576 17292 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.005s
15:10:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.812879 17307 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.814301 17292 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
15:10:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.814456 17292 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/data,/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/wal
15:10:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "799c4434881c4cb889a6fbbd7e6ae196"
15:10:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:02 on dist-test-slave-kz7d"
15:10:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.814651 17292 fs_report.cc:383] FS layout report
15:10:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/wal
15:10:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/wal
15:10:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/data/data
15:10:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:02.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.844242 17292 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:10:02.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.845216 17292 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:02.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.845538 17292 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:02.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.846565 17292 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:10:02.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.848436 17292 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:10:02.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.848510 17292 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:10:02.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.848624 17292 ts_tablet_manager.cc:610] Registered 0 tablets
15:10:02.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.848676 17292 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:10:02.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.885330 17292 rpc_server.cc:307] RPC server started. Bound to: 127.16.123.66:40077
15:10:02.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.885360 17419 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.123.66:40077 every 8 connection(s)
15:10:02.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.886675 17292 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/data/info.pb
15:10:02.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.887262 16877 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu as pid 17292
15:10:02.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.887404 16877 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-1/wal/instance
15:10:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.894269 16877 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/wal
15:10:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/data
15:10:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/logs
15:10:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/data/info.pb
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.123.67:0
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.123.67
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.123.67
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.123.126:46353,127.16.123.125:33429,127.16.123.124:35659
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.123.84:35073
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:10:02.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.917088 17422 heartbeater.cc:347] Connected to a master server at 127.16.123.125:33429
15:10:02.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.917310 17421 heartbeater.cc:347] Connected to a master server at 127.16.123.126:46353
15:10:02.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.919569 17421 heartbeater.cc:464] Registering TS with master...
15:10:02.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.920480 17421 heartbeater.cc:511] Master 127.16.123.126:46353 requested a full tablet report, sending...
15:10:02.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.917480 17420 heartbeater.cc:347] Connected to a master server at 127.16.123.124:35659
15:10:02.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.917799 17422 heartbeater.cc:464] Registering TS with master...
15:10:02.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.921412 17420 heartbeater.cc:464] Registering TS with master...
15:10:02.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.922120 17420 heartbeater.cc:511] Master 127.16.123.124:35659 requested a full tablet report, sending...
15:10:02.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.922766 17422 heartbeater.cc:511] Master 127.16.123.125:33429 requested a full tablet report, sending...
15:10:02.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.922995 16919 ts_manager.cc:194] Registered new tserver with Master: 799c4434881c4cb889a6fbbd7e6ae196 (127.16.123.66:40077)
15:10:02.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.923517 17061 ts_manager.cc:194] Registered new tserver with Master: 799c4434881c4cb889a6fbbd7e6ae196 (127.16.123.66:40077)
15:10:02.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.924178 16990 ts_manager.cc:194] Registered new tserver with Master: 799c4434881c4cb889a6fbbd7e6ae196 (127.16.123.66:40077)
15:10:02.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:02.924728 16919 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.123.66:48815
15:10:03.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:03.243330 17427 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:03.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:03.243598 17427 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:03.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:03.243673 17427 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:03.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:03.251282 17427 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:03.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:03.251458 17427 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.123.67
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.260344 17427 tablet_server_runner.cc:78] Tablet server non-default flags:
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.123.84:35073
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/data
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/wal
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.123.67:0
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/data/info.pb
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.123.67
15:10:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.123.126:46353,127.16.123.125:33429,127.16.123.124:35659
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.123.67
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/logs
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 13:46:26 UTC on ccycloud.quasar-xkqhuq.root.comops.site
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676080
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:10:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.261592 17427 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:03.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.263211 17427 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:10:03.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:03.275923 17432 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:03.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:03.275942 17435 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:03.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:03.276022 17433 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:03.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.277607 17427 server_base.cc:1034] running on GCE node
15:10:03.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.278142 17427 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:03.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.279054 17427 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.280275 17427 hybrid_clock.cc:648] HybridClock initialized: now 1790521803280211 us; error 40 us; skew 500 ppm
15:10:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.280529 17427 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:03.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.283058 17427 webserver.cc:460] Webserver started at http://127.16.123.67:46449/ using document root <none> and password file <none>
15:10:03.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.283641 17427 fs_manager.cc:359] Metadata directory not provided
15:10:03.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.283732 17427 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:03.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.283948 17427 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:10:03.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.286360 17427 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/data/instance:
15:10:03.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "183a6ef44e2d46589afb619dd7f4bcda"
15:10:03.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:03 on dist-test-slave-kz7d"
15:10:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.287021 17427 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/wal/instance:
15:10:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "183a6ef44e2d46589afb619dd7f4bcda"
15:10:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:03 on dist-test-slave-kz7d"
15:10:03.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.292004 17427 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
15:10:03.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.295665 17441 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.297022 17427 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.297159 17427 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/data,/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/wal
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "183a6ef44e2d46589afb619dd7f4bcda"
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:03 on dist-test-slave-kz7d"
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.297354 17427 fs_report.cc:383] FS layout report
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/wal
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/wal
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/data/data
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:03.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:03.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.311237 17427 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:10:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.312404 17427 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.312634 17427 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:03.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.313527 17427 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:10:03.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.315006 17427 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:10:03.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.315074 17427 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:10:03.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.315153 17427 ts_tablet_manager.cc:610] Registered 0 tablets
15:10:03.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.315186 17427 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:10:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.347561 17427 rpc_server.cc:307] RPC server started. Bound to: 127.16.123.67:40119
15:10:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.347602 17553 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.123.67:40119 every 8 connection(s)
15:10:03.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.348778 17427 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/data/info.pb
15:10:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.357381 16877 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu as pid 17427
15:10:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.357549 16877 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster3649081935668237144/ts-2/wal/instance
15:10:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.367420 17554 heartbeater.cc:347] Connected to a master server at 127.16.123.124:35659
15:10:03.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.367650 17555 heartbeater.cc:347] Connected to a master server at 127.16.123.126:46353
15:10:03.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.367759 17555 heartbeater.cc:464] Registering TS with master...
15:10:03.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.367763 17554 heartbeater.cc:464] Registering TS with master...
15:10:03.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.367812 17556 heartbeater.cc:347] Connected to a master server at 127.16.123.125:33429
15:10:03.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.367978 17556 heartbeater.cc:464] Registering TS with master...
15:10:03.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.368741 17555 heartbeater.cc:511] Master 127.16.123.126:46353 requested a full tablet report, sending...
15:10:03.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.368767 17556 heartbeater.cc:511] Master 127.16.123.125:33429 requested a full tablet report, sending...
15:10:03.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.368793 17554 heartbeater.cc:511] Master 127.16.123.124:35659 requested a full tablet report, sending...
15:10:03.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.370323 16919 ts_manager.cc:194] Registered new tserver with Master: 183a6ef44e2d46589afb619dd7f4bcda (127.16.123.67:40119)
15:10:03.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.370572 17061 ts_manager.cc:194] Registered new tserver with Master: 183a6ef44e2d46589afb619dd7f4bcda (127.16.123.67:40119)
15:10:03.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.370574 16990 ts_manager.cc:194] Registered new tserver with Master: 183a6ef44e2d46589afb619dd7f4bcda (127.16.123.67:40119)
15:10:03.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.371315 16919 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.123.67:53385
15:10:03.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.383803 16877 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
15:10:03.385 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:10:03.386 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:10:03.397 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.16.123.126"
port: 46353
}
bound_http_address {
host: "127.16.123.126"
port: 35445
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.16.123.125"
port: 33429
}
bound_http_address {
host: "127.16.123.125"
port: 46667
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.16.123.124"
port: 35659
}
bound_http_address {
host: "127.16.123.124"
port: 36469
}
}
}
15:10:03.401 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:10:03.405 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.16.123.65"
port: 45861
}
bound_http_address {
host: "127.16.123.65"
port: 37151
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.16.123.66"
port: 40077
}
bound_http_address {
host: "127.16.123.66"
port: 44391
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.16.123.67"
port: 40119
}
bound_http_address {
host: "127.16.123.67"
port: 46449
}
}
}
15:10:03.405 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:10:03.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.444458 17284 heartbeater.cc:503] Master 127.16.123.126:46353 was elected leader, sending a full tablet report...
15:10:03.575 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:10:03.690 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskY4X0Fv/test-tmp
15:10:03.691 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
15:10:03.692 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskY4X0Fv/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
15:10:03.695 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
15:10:03.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:03.929872 17421 heartbeater.cc:503] Master 127.16.123.126:46353 was elected leader, sending a full tablet report...
15:10:04.054 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:04.054 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:10:04.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:04.056499 17562 mini_oidc.cc:152] Starting JWKS server
15:10:04.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:04.058959 17562 webserver.cc:460] Webserver started at http://127.0.0.1:42597/ using document root <none> and password file <none>
15:10:04.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:04.059154 17562 mini_oidc.cc:189] Starting OIDC Discovery server
15:10:04.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:04.059695 17562 webserver.cc:460] Webserver started at http://127.0.0.1:44283/ using document root <none> and password file <none>
15:10:04.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
15:10:04.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/principal' for realm 'KRBTEST.COM',
15:10:04.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
15:10:04.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:04 dist-test-slave-kz7d krb5kdc[17573](info): setting up network...
15:10:04.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
15:10:04.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:04 dist-test-slave-kz7d krb5kdc[17573](info): set up 2 sockets
15:10:04.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:04 dist-test-slave-kz7d krb5kdc[17573](info): commencing operation
15:10:04.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
15:10:04.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:04.373634 17555 heartbeater.cc:503] Master 127.16.123.126:46353 was elected leader, sending a full tablet report...
15:10:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.051404 17562 mini_kdc.cc:121] Time spent starting KDC: real 0.983s user 0.005s sys 0.008s
15:10:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
15:10:05.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:10:05.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
15:10:05.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
15:10:05.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:10:05.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
15:10:05.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
15:10:05.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:10:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
15:10:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:10:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/test-user.keytab.
15:10:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/test-user.keytab.
15:10:05.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:05 dist-test-slave-kz7d krb5kdc[17573](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521805, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:05.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:10:05.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T15:10:05Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
15:10:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T15:10:05Z Disabled control of system clock
15:10:05.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.38.190@KRBTEST.COM; defaulting to no policy
15:10:05.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:05.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.38.190@KRBTEST.COM" created.
15:10:05.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:05.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.38.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.190.keytab.
15:10:05.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.38.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.190.keytab.
15:10:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.205291 17562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal
15:10:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data
15:10:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/logs
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data/info.pb
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.38.190:34925
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.38.190
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.190.keytab
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.38.190
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.38.148:33685
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.38.190:34925,127.17.38.189:33571,127.17.38.188:41141
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42597/jwks.json
15:10:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:05.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.521082 17586 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:05.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.521420 17586 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:05.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.521553 17586 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:05.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.529177 17586 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:05.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.529321 17586 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:05.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.529415 17586 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:05.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.529487 17586 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:05.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.529534 17586 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:05.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.529624 17586 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:05.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.529696 17586 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42597/jwks.json
15:10:05.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.539631 17586 master_runner.cc:386] Master server non-default flags:
15:10:05.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.38.148:33685
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.38.190:34925,127.17.38.189:33571,127.17.38.188:41141
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.190.keytab
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.38.190
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.38.190:34925
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42597/jwks.json
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data/info.pb
15:10:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.38.190
15:10:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/logs
15:10:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:10:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:10:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 13:46:26 UTC on ccycloud.quasar-xkqhuq.root.comops.site
15:10:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676080
15:10:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:10:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.541229 17586 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.542780 17586 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:10:05.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.554353 17591 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.554376 17592 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.554517 17594 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.555076 17586 server_base.cc:1034] running on GCE node
15:10:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.555807 17586 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:05.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.557176 17586 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:05.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.558429 17586 hybrid_clock.cc:648] HybridClock initialized: now 1790521805558374 us; error 38 us; skew 500 ppm
15:10:05.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.558797 17586 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:05 dist-test-slave-kz7d krb5kdc[17573](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521805, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:05.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.570519 17586 init.cc:377] Logged in from keytab as kudu/127.17.38.190@KRBTEST.COM (short username kudu)
15:10:05.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.571997 17586 webserver.cc:460] Webserver started at http://127.17.38.190:43385/ using document root <none> and password file <none>
15:10:05.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.572713 17586 fs_manager.cc:359] Metadata directory not provided
15:10:05.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.572829 17586 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:05.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.573137 17586 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:10:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.575817 17586 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data/instance:
15:10:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63c7afaba27f4ab083c3d882f8cd3e49"
15:10:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:05 on dist-test-slave-kz7d"
15:10:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.576566 17586 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal/instance:
15:10:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63c7afaba27f4ab083c3d882f8cd3e49"
15:10:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:05 on dist-test-slave-kz7d"
15:10:05.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.582028 17586 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.001s
15:10:05.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.585271 17601 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:05.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.586621 17586 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
15:10:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.586790 17586 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data,/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal
15:10:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63c7afaba27f4ab083c3d882f8cd3e49"
15:10:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:05 on dist-test-slave-kz7d"
15:10:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.586980 17586 fs_report.cc:383] FS layout report
15:10:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal
15:10:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal
15:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data/data
15:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.611925 17586 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:10:05.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.616570 17586 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:05.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.616832 17586 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:05.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.634647 17586 rpc_server.cc:307] RPC server started. Bound to: 127.17.38.190:34925
15:10:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.634675 17652 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.38.190:34925 every 8 connection(s)
15:10:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.635977 17586 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data/info.pb
15:10:05.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.638679 17562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu as pid 17586
15:10:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.638957 17562 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal/instance
15:10:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.640414 17653 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:10:05.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.647295 17653 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } has no permanent_uuid. Determining permanent_uuid...
15:10:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.38.189@KRBTEST.COM; defaulting to no policy
15:10:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.38.189@KRBTEST.COM" created.
15:10:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:05 dist-test-slave-kz7d krb5kdc[17573](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521805, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.190@KRBTEST.COM for kudu/127.17.38.190@KRBTEST.COM
15:10:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.671672 17653 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } has no permanent_uuid. Determining permanent_uuid...
15:10:05.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.673305 17603 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111)
15:10:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.674804 17653 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.189:33571: Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111)
15:10:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.38.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.189.keytab.
15:10:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.38.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.189.keytab.
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.676849 17562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/logs
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data/info.pb
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:05.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.38.189:33571
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.38.189
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.189.keytab
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.38.189
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.38.148:33685
15:10:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.38.190:34925,127.17.38.189:33571,127.17.38.188:41141
15:10:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42597/jwks.json
15:10:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:05.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.724112 17653 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } attempt: 1
15:10:05.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.725981 17653 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.189:33571: Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111)
15:10:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.794247 17653 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } attempt: 2
15:10:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.796368 17653 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.189:33571: Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111)
15:10:05.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:05.887663 17653 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } attempt: 3
15:10:05.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.889598 17653 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.189:33571: Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111)
15:10:05.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.997308 17660 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:05.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.997601 17660 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:05.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:05.997691 17660 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.005168 17660 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.005312 17660 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.005362 17660 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.005409 17660 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.005442 17660 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.005498 17660 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.005528 17660 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42597/jwks.json
15:10:06.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.015628 17660 master_runner.cc:386] Master server non-default flags:
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.38.148:33685
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.38.190:34925,127.17.38.189:33571,127.17.38.188:41141
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.189.keytab
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.38.189
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.38.189:33571
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42597/jwks.json
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:06.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data/info.pb
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.38.189
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/logs
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 13:46:26 UTC on ccycloud.quasar-xkqhuq.root.comops.site
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676080
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:10:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.016955 17660 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.018738 17660 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:10:06.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.030107 17666 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:06.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.030576 17665 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:06.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.030543 17668 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:06.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.031611 17660 server_base.cc:1034] running on GCE node
15:10:06.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.032442 17660 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.032806 17653 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } attempt: 4
15:10:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.034418 17660 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.034840 17653 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.189:33571: Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111)
15:10:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.035638 17660 hybrid_clock.cc:648] HybridClock initialized: now 1790521806035585 us; error 44 us; skew 500 ppm
15:10:06.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.035949 17660 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:06.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521806, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:06.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.039642 17660 init.cc:377] Logged in from keytab as kudu/127.17.38.189@KRBTEST.COM (short username kudu)
15:10:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.041489 17660 webserver.cc:460] Webserver started at http://127.17.38.189:39367/ using document root <none> and password file <none>
15:10:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.042161 17660 fs_manager.cc:359] Metadata directory not provided
15:10:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.042260 17660 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.042517 17660 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:10:06.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.045266 17660 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data/instance:
15:10:06.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "75c721a7e130478dae871c7bc5bef5b8"
15:10:06.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:06 on dist-test-slave-kz7d"
15:10:06.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.045972 17660 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal/instance:
15:10:06.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "75c721a7e130478dae871c7bc5bef5b8"
15:10:06.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:06 on dist-test-slave-kz7d"
15:10:06.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.050930 17660 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
15:10:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.054183 17675 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.055871 17660 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.056067 17660 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data,/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "75c721a7e130478dae871c7bc5bef5b8"
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:06 on dist-test-slave-kz7d"
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.056274 17660 fs_report.cc:383] FS layout report
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data/data
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:06.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.081318 17660 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:10:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.085532 17660 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.085769 17660 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.105007 17660 rpc_server.cc:307] RPC server started. Bound to: 127.17.38.189:33571
15:10:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.105010 17726 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.38.189:33571 every 8 connection(s)
15:10:06.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.106462 17660 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data/info.pb
15:10:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.109999 17727 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:10:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.111275 17562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu as pid 17660
15:10:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.111388 17562 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal/instance
15:10:06.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.116693 17727 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } has no permanent_uuid. Determining permanent_uuid...
15:10:06.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.38.188@KRBTEST.COM; defaulting to no policy
15:10:06.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:06.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.38.188@KRBTEST.COM" created.
15:10:06.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521806, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.189@KRBTEST.COM for kudu/127.17.38.190@KRBTEST.COM
15:10:06.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.141479 17727 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } has no permanent_uuid. Determining permanent_uuid...
15:10:06.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:06.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.38.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.188.keytab.
15:10:06.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.38.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.188.keytab.
15:10:06.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.149570 17562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:06.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:06.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/wal
15:10:06.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data
15:10:06.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:06.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:06.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:06.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:06.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/logs
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data/info.pb
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.38.188:41141
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.38.188
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.188.keytab
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.38.188
15:10:06.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.38.148:33685
15:10:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.38.190:34925,127.17.38.189:33571,127.17.38.188:41141
15:10:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42597/jwks.json
15:10:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:06.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521806, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.189@KRBTEST.COM for kudu/127.17.38.189@KRBTEST.COM
15:10:06.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.160475 17727 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } has no permanent_uuid. Determining permanent_uuid...
15:10:06.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.162099 17676 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: connect: Connection refused (error 111)
15:10:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.164593 17727 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.188:41141: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: connect: Connection refused (error 111)
15:10:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.213876 17727 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } attempt: 1
15:10:06.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.215843 17727 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.188:41141: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: connect: Connection refused (error 111)
15:10:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.284102 17727 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } attempt: 2
15:10:06.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.285987 17727 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.188:41141: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: connect: Connection refused (error 111)
15:10:06.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.334116 17653 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } attempt: 5
15:10:06.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521805, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.190@KRBTEST.COM for kudu/127.17.38.189@KRBTEST.COM
15:10:06.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.348563 17653 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } has no permanent_uuid. Determining permanent_uuid...
15:10:06.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.350337 17653 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.188:41141: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: connect: Connection refused (error 111)
15:10:06.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.377295 17727 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } attempt: 3
15:10:06.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.379133 17727 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.188:41141: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: connect: Connection refused (error 111)
15:10:06.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.401507 17653 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } attempt: 1
15:10:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.403678 17653 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.188:41141: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: connect: Connection refused (error 111)
15:10:06.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.471860 17653 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } attempt: 2
15:10:06.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.473624 17653 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.188:41141: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: connect: Connection refused (error 111)
15:10:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.474583 17733 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.474905 17733 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:06.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.475042 17733 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.483662 17733 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.483748 17733 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.483795 17733 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.483840 17733 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.483881 17733 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.483933 17733 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:06.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.483963 17733 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42597/jwks.json
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.493847 17733 master_runner.cc:386] Master server non-default flags:
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.38.148:33685
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/wal
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.38.190:34925,127.17.38.189:33571,127.17.38.188:41141
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.188.keytab
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.38.188
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.38.188:41141
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42597/jwks.json
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data/info.pb
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.38.188
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/logs
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 13:46:26 UTC on ccycloud.quasar-xkqhuq.root.comops.site
15:10:06.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676080
15:10:06.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:10:06.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.495030 17733 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:06.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.496194 17733 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:10:06.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.507069 17742 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:06.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.507710 17739 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:06.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.507941 17733 server_base.cc:1034] running on GCE node
15:10:06.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.507097 17740 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:06.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.509334 17733 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:06.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.510593 17733 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:06.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.511792 17733 hybrid_clock.cc:648] HybridClock initialized: now 1790521806511746 us; error 39 us; skew 500 ppm
15:10:06.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.512127 17733 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:06.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521806, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:06.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.515197 17733 init.cc:377] Logged in from keytab as kudu/127.17.38.188@KRBTEST.COM (short username kudu)
15:10:06.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.516512 17733 webserver.cc:460] Webserver started at http://127.17.38.188:38927/ using document root <none> and password file <none>
15:10:06.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.517102 17733 fs_manager.cc:359] Metadata directory not provided
15:10:06.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.517246 17733 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:06.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.517506 17733 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:10:06.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.520021 17733 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data/instance:
15:10:06.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06f2b15c049a444cb78ebe625650a148"
15:10:06.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:06 on dist-test-slave-kz7d"
15:10:06.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.520634 17733 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/wal/instance:
15:10:06.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06f2b15c049a444cb78ebe625650a148"
15:10:06.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:06 on dist-test-slave-kz7d"
15:10:06.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.522320 17727 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } attempt: 4
15:10:06.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.524016 17727 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.188:41141: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: connect: Connection refused (error 111)
15:10:06.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.525784 17733 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.003s
15:10:06.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.529266 17749 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.530587 17733 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
15:10:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.530714 17733 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data,/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/wal
15:10:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06f2b15c049a444cb78ebe625650a148"
15:10:06.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:06 on dist-test-slave-kz7d"
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.530854 17733 fs_report.cc:383] FS layout report
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/wal
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/wal
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data/data
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:06.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.562839 17733 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:10:06.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.566596 17733 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:06.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.566823 17733 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:06.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.579819 17653 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } attempt: 3
15:10:06.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.581806 17653 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.188:41141: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: connect: Connection refused (error 111)
15:10:06.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.585850 17733 rpc_server.cc:307] RPC server started. Bound to: 127.17.38.188:41141
15:10:06.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.585974 17800 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.38.188:41141 every 8 connection(s)
15:10:06.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.587024 17733 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data/info.pb
15:10:06.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.590406 17801 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:10:06.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.594471 17562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu as pid 17733
15:10:06.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.594627 17562 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/wal/instance
15:10:06.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.597155 17801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } has no permanent_uuid. Determining permanent_uuid...
15:10:06.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521806, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.188@KRBTEST.COM for kudu/127.17.38.190@KRBTEST.COM
15:10:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521805, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.38.188@KRBTEST.COM
15:10:06.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.622265 17801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } has no permanent_uuid. Determining permanent_uuid...
15:10:06.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521806, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.188@KRBTEST.COM for kudu/127.17.38.189@KRBTEST.COM
15:10:06.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.636705 17801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } has no permanent_uuid. Determining permanent_uuid...
15:10:06.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521806, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.188@KRBTEST.COM for kudu/127.17.38.188@KRBTEST.COM
15:10:06.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521805, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.38.190@KRBTEST.COM
15:10:06.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.655228 17801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: Bootstrap starting.
15:10:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.658416 17801 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: Neither blocks nor log segments found. Creating new log.
15:10:06.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.659346 17801 log.cc:824] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: Log is configured to *not* fsync() on all Append() calls
15:10:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521805, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.38.189@KRBTEST.COM
15:10:06.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.662259 17801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: No bootstrap required, opened a new log
15:10:06.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.670398 17562 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
15:10:06.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.670557 17801 raft_consensus.cc:348] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } } peers { permanent_uuid: "75c721a7e130478dae871c7bc5bef5b8" member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } } peers { permanent_uuid: "06f2b15c049a444cb78ebe625650a148" member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } }
15:10:06.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.670830 17801 raft_consensus.cc:374] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:10:06.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.670897 17801 raft_consensus.cc:729] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 06f2b15c049a444cb78ebe625650a148, State: Initialized, Role: FOLLOWER
15:10:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.671356 17801 consensus_queue.cc:260] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } } peers { permanent_uuid: "75c721a7e130478dae871c7bc5bef5b8" member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } } peers { permanent_uuid: "06f2b15c049a444cb78ebe625650a148" member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } }
15:10:06.672 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:06.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.672775 17807 sys_catalog.cc:455] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } } peers { permanent_uuid: "75c721a7e130478dae871c7bc5bef5b8" member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } } peers { permanent_uuid: "06f2b15c049a444cb78ebe625650a148" member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } } }
15:10:06.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.673756 17807 sys_catalog.cc:458] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [sys.catalog]: This master's current role is: FOLLOWER
15:10:06.674 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
15:10:06.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.674563 17801 sys_catalog.cc:564] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [sys.catalog]: configured and running, proceeding with master startup.
15:10:06.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
15:10:06.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
15:10:06.690 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.17.38.190"
port: 34925
}
bound_http_address {
host: "127.17.38.190"
port: 43385
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.17.38.189"
port: 33571
}
bound_http_address {
host: "127.17.38.189"
port: 39367
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.17.38.188"
port: 41141
}
bound_http_address {
host: "127.17.38.188"
port: 38927
}
}
}
15:10:06.690 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:10:06.690 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:06.690 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:10:06.692 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:10:06.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.694757 17818 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:06.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.695056 17818 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:06.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.685171 17733 master_runner.cc:186] Error getting master registration for 127.17.38.190:34925: OK, instance_id { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" instance_seqno: 1790521805629319 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:10:06.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.697464 17733 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:06.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521806, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:06.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:10:06.719 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:06.721 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5cc
15:10:06.726 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:10:06.727 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5cc
15:10:06.728 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:10:06.732 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
15:10:06.733 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:10:06.734 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:10:06.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521806, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:10:06.757 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:06.757 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5cc
15:10:06.759 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:10:06.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.759027 17653 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } attempt: 4
15:10:06.760 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.38.188:41141
15:10:06.761 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:10:06.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.762275 17562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu pid 17733
15:10:06.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.763756 17796 generic_service.cc:196] Checking for leaks (request via RPC)
15:10:06.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.823225 17727 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } attempt: 5
15:10:06.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.857095 17755 tablet.cc:2369] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:10:06.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.860570 17562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu with pid 17733
15:10:06.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:06 dist-test-slave-kz7d krb5kdc[17573](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521805, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.190@KRBTEST.COM for kudu/127.17.38.188@KRBTEST.COM
15:10:06.874 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:06.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.874063 17730 negotiation.cc:337] Failed RPC negotiation. Trace:
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.823919 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.17.38.188:41141 (local address 127.0.0.1:46548)
15:10:06.875 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.38.190:34925
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.824094 (+ 175us) negotiation.cc:107] Waiting for socket to connect
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.824110 (+ 16us) client_negotiation.cc:174] Beginning negotiation
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.824205 (+ 95us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.859780 (+ 35575us) client_negotiation.cc:271] Received NEGOTIATE NegotiatePB response
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.859787 (+ 7us) client_negotiation.cc:394] Received NEGOTIATE response from server
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.860214 (+ 427us) client_negotiation.cc:189] Negotiated authn=SASL
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.860407 (+ 193us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.860419 (+ 12us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.872907 (+ 12488us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: BlockingRecv error: recv got EOF from 127.17.38.188:41141 (error 108)
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":56}
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.875059 17656 negotiation.cc:337] Failed RPC negotiation. Trace:
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.759661 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.17.38.188:41141 (local address 127.0.0.1:46538)
15:10:06.875 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.759875 (+ 214us) negotiation.cc:107] Waiting for socket to connect
15:10:06.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.759897 (+ 22us) client_negotiation.cc:174] Beginning negotiation
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.760017 (+ 120us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.760534 (+ 517us) client_negotiation.cc:271] Received NEGOTIATE NegotiatePB response
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.760539 (+ 5us) client_negotiation.cc:394] Received NEGOTIATE response from server
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.761003 (+ 464us) client_negotiation.cc:189] Negotiated authn=SASL
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.761225 (+ 222us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.761237 (+ 12us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.762708 (+ 1471us) client_negotiation.cc:271] Received TLS_HANDSHAKE NegotiatePB response
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.762714 (+ 6us) client_negotiation.cc:527] Received TLS_HANDSHAKE response from server
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.763918 (+ 1204us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.763935 (+ 17us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.858939 (+ 95004us) client_negotiation.cc:271] Received TLS_HANDSHAKE NegotiatePB response
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.858945 (+ 6us) client_negotiation.cc:527] Received TLS_HANDSHAKE response from server
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.859146 (+ 201us) client_negotiation.cc:548] Negotiated auth-only TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 Kx=ECDH Au=RSA Enc=AESGCM(128) Mac=AEAD
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.859837 (+ 691us) client_negotiation.cc:623] Initiating SASL GSSAPI handshake
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.859863 (+ 26us) client_negotiation.cc:656] calling sasl_client_start()
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.866743 (+ 6880us) client_negotiation.cc:252] Sending SASL_INITIATE NegotiatePB request
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0927 15:10:06.874362 (+ 7619us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":63}
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.876150 17562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu pid 17586
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.876542 17727 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.188:41141: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: connect: Connection refused (error 111)
15:10:06.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:06.876710 17653 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.38.188:41141: Network error: Client connection negotiation failed: client connection to 127.17.38.188:41141: connect: Connection refused (error 111)
15:10:06.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.877307 17648 generic_service.cc:196] Checking for leaks (request via RPC)
15:10:06.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.955721 17562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu with pid 17586
15:10:06.967 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:06.967 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.38.189:33571
15:10:06.967 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:10:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.968115 17562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu pid 17660
15:10:06.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:06.969889 17722 generic_service.cc:196] Checking for leaks (request via RPC)
15:10:07.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.044545 17562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu with pid 17660
15:10:07.057 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:07.057 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.38.188:41141
15:10:07.058 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:10:07.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.059789 17562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/wal
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/logs
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data/info.pb
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.38.188:41141
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.38.188
15:10:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38927
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.188.keytab
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.38.188
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.38.148:33685
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.38.190:34925,127.17.38.189:33571,127.17.38.188:41141
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42597/jwks.json
15:10:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:07.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.363989 17828 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:07.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.364246 17828 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:07.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.364338 17828 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:07.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.371531 17828 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:07.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.371620 17828 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:07.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.371666 17828 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:07.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.371712 17828 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:07.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.371747 17828 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:07.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.371800 17828 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:07.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.371835 17828 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42597/jwks.json
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.381870 17828 master_runner.cc:386] Master server non-default flags:
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.38.148:33685
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/wal
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.38.190:34925,127.17.38.189:33571,127.17.38.188:41141
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.188.keytab
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.38.188
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.38.188:41141
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42597/jwks.json
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data/info.pb
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.38.188
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38927
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/logs
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 13:46:26 UTC on ccycloud.quasar-xkqhuq.root.comops.site
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676080
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:10:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.383088 17828 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:07.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.384243 17828 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:10:07.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.395056 17833 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:07.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.395128 17834 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:07.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.395155 17836 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:07.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.396149 17828 server_base.cc:1034] running on GCE node
15:10:07.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.396832 17828 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:07.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.398223 17828 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:07.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.399459 17828 hybrid_clock.cc:648] HybridClock initialized: now 1790521807399405 us; error 56 us; skew 500 ppm
15:10:07.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.399760 17828 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:07.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:07 dist-test-slave-kz7d krb5kdc[17573](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521807, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:07.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.402776 17828 init.cc:377] Logged in from keytab as kudu/127.17.38.188@KRBTEST.COM (short username kudu)
15:10:07.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.404526 17828 webserver.cc:460] Webserver started at http://127.17.38.188:38927/ using document root <none> and password file <none>
15:10:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.405210 17828 fs_manager.cc:359] Metadata directory not provided
15:10:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.405345 17828 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:07.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.410713 17828 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.003s sys 0.001s
15:10:07.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.413244 17843 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:07.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.414420 17828 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
15:10:07.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.414561 17828 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data,/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/wal
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06f2b15c049a444cb78ebe625650a148"
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:06 on dist-test-slave-kz7d"
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.415529 17828 fs_report.cc:383] FS layout report
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/wal
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/wal
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data/data
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:07.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.430752 17828 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:10:07.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.434633 17828 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:07.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.434896 17828 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:07.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.452489 17828 rpc_server.cc:307] RPC server started. Bound to: 127.17.38.188:41141
15:10:07.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.452548 17894 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.38.188:41141 every 8 connection(s)
15:10:07.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.453889 17828 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-2/data/info.pb
15:10:07.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.460114 17895 sys_catalog.cc:263] Verifying existing consensus state
15:10:07.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.463404 17895 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: Bootstrap starting.
15:10:07.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.463912 17562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu as pid 17828
15:10:07.464 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:07.464 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.38.190:34925
15:10:07.464 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:10:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.465564 17562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal
15:10:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data
15:10:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/logs
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data/info.pb
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.38.190:34925
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.38.190
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43385
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.190.keytab
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.38.190
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.38.148:33685
15:10:07.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:07.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:07.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:07.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:07.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.38.190:34925,127.17.38.189:33571,127.17.38.188:41141
15:10:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42597/jwks.json
15:10:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:07.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.482148 17895 log.cc:824] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: Log is configured to *not* fsync() on all Append() calls
15:10:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.484620 17895 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:10:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.485057 17895 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: Bootstrap complete.
15:10:07.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.493707 17895 raft_consensus.cc:348] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } } peers { permanent_uuid: "75c721a7e130478dae871c7bc5bef5b8" member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } } peers { permanent_uuid: "06f2b15c049a444cb78ebe625650a148" member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } }
15:10:07.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.494556 17895 raft_consensus.cc:374] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:10:07.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.494620 17895 raft_consensus.cc:729] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 06f2b15c049a444cb78ebe625650a148, State: Initialized, Role: FOLLOWER
15:10:07.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.495185 17895 consensus_queue.cc:260] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } } peers { permanent_uuid: "75c721a7e130478dae871c7bc5bef5b8" member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } } peers { permanent_uuid: "06f2b15c049a444cb78ebe625650a148" member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } }
15:10:07.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.495980 17899 sys_catalog.cc:455] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } } peers { permanent_uuid: "75c721a7e130478dae871c7bc5bef5b8" member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } } peers { permanent_uuid: "06f2b15c049a444cb78ebe625650a148" member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } } }
15:10:07.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.496361 17899 sys_catalog.cc:458] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [sys.catalog]: This master's current role is: FOLLOWER
15:10:07.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.496999 17895 sys_catalog.cc:564] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [sys.catalog]: configured and running, proceeding with master startup.
15:10:07.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.505656 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.505991 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:07.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.507407 17828 master_runner.cc:186] Error getting master registration for 127.17.38.190:34925: Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111),
15:10:07.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.516970 17828 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:07.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.583827 17899 raft_consensus.cc:482] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:07.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.584136 17899 raft_consensus.cc:504] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } } peers { permanent_uuid: "75c721a7e130478dae871c7bc5bef5b8" member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } } peers { permanent_uuid: "06f2b15c049a444cb78ebe625650a148" member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } }
15:10:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.586191 17899 leader_election.cc:288] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925), 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571)
15:10:07.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.587121 17844 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111)
15:10:07.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.599783 17845 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571): Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111)
15:10:07.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.600220 17844 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925): Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111)
15:10:07.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.600548 17844 leader_election.cc:302] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 06f2b15c049a444cb78ebe625650a148; no voters: 63c7afaba27f4ab083c3d882f8cd3e49, 75c721a7e130478dae871c7bc5bef5b8
15:10:07.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.600944 17899 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:10:07.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.686802 17849 tablet.cc:2369] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:10:07.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.781781 17897 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:07.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.782053 17897 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:07.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.782142 17897 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.789116 17897 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.789244 17897 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.789307 17897 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.789356 17897 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.789391 17897 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.789444 17897 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.789474 17897 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42597/jwks.json
15:10:07.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.799419 17897 master_runner.cc:386] Master server non-default flags:
15:10:07.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:07.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.38.148:33685
15:10:07.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:07.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:07.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.38.190:34925,127.17.38.189:33571,127.17.38.188:41141
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.190.keytab
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.38.190
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.38.190:34925
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42597/jwks.json
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:07.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data/info.pb
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.38.190
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43385
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/logs
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:10:07.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:10:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 13:46:26 UTC on ccycloud.quasar-xkqhuq.root.comops.site
15:10:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676080
15:10:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:10:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.800796 17897 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.802145 17897 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:10:07.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.814941 17922 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:07.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.814878 17920 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:07.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:07.814901 17919 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.818801 17897 server_base.cc:1034] running on GCE node
15:10:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.819613 17897 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:07.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.821080 17897 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:07.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.822284 17897 hybrid_clock.cc:648] HybridClock initialized: now 1790521807822232 us; error 62 us; skew 500 ppm
15:10:07.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.822588 17897 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:07.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:07 dist-test-slave-kz7d krb5kdc[17573](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521807, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:07.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.825599 17897 init.cc:377] Logged in from keytab as kudu/127.17.38.190@KRBTEST.COM (short username kudu)
15:10:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.827327 17897 webserver.cc:460] Webserver started at http://127.17.38.190:43385/ using document root <none> and password file <none>
15:10:07.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.827998 17897 fs_manager.cc:359] Metadata directory not provided
15:10:07.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.828120 17897 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:07.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.833280 17897 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.004s sys 0.000s
15:10:07.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.835824 17929 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.837127 17897 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.837302 17897 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data,/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "63c7afaba27f4ab083c3d882f8cd3e49"
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:05 on dist-test-slave-kz7d"
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.838172 17897 fs_report.cc:383] FS layout report
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data/data
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:07.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.865901 17897 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:10:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.869624 17897 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.869840 17897 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:07.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.887126 17897 rpc_server.cc:307] RPC server started. Bound to: 127.17.38.190:34925
15:10:07.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.887146 17980 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.38.190:34925 every 8 connection(s)
15:10:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.888470 17897 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/data/info.pb
15:10:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.893332 17562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu as pid 17897
15:10:07.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:07.894 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.38.189:33571
15:10:07.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:10:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.895470 17981 sys_catalog.cc:263] Verifying existing consensus state
15:10:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.895491 17562 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu
15:10:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal
15:10:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data
15:10:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:07.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/logs
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data/info.pb
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.38.189:33571
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.38.189
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39367
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.189.keytab
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.38.189
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.38.148:33685
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.38.190:34925,127.17.38.189:33571,127.17.38.188:41141
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42597/jwks.json
15:10:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 15:10:07.895920 17981 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:10:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.896121 17897 master.cc:537] Master@127.17.38.190:34925 shutting down...
15:10:07.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:07.919245 17897 master.cc:552] Master@127.17.38.190:34925 shutdown complete.
15:10:07.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:10:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.231880 17983 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.232163 17983 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.232295 17983 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.239574 17983 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.239668 17983 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.239737 17983 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.239801 17983 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:08.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.239853 17983 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:08.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.239930 17983 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:08.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.239979 17983 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42597/jwks.json
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.250061 17983 master_runner.cc:386] Master server non-default flags:
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.38.148:33685
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.38.190:34925,127.17.38.189:33571,127.17.38.188:41141
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/krb5kdc/kudu_127.17.38.189.keytab
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.38.189
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.38.189:33571
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:08.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42597/jwks.json
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data/info.pb
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.38.189
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39367
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/logs
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 13:46:26 UTC on ccycloud.quasar-xkqhuq.root.comops.site
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676080
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
15:10:08.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.251679 17983 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:08.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.253521 17983 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:10:08.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.265130 17989 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:08.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.265101 17990 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:08.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.265101 17992 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:08.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.265691 17983 server_base.cc:1034] running on GCE node
15:10:08.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.266433 17983 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:08.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.267791 17983 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.269023 17983 hybrid_clock.cc:648] HybridClock initialized: now 1790521808268964 us; error 38 us; skew 500 ppm
15:10:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.269407 17983 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:08 dist-test-slave-kz7d krb5kdc[17573](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521808, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:08.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.272362 17983 init.cc:377] Logged in from keytab as kudu/127.17.38.189@KRBTEST.COM (short username kudu)
15:10:08.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.274470 17983 webserver.cc:460] Webserver started at http://127.17.38.189:39367/ using document root <none> and password file <none>
15:10:08.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.275287 17983 fs_manager.cc:359] Metadata directory not provided
15:10:08.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.275434 17983 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:08.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.280478 17983 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.005s sys 0.000s
15:10:08.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.283295 17999 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:08.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.284626 17983 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
15:10:08.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.284761 17983 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data,/tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal
15:10:08.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "75c721a7e130478dae871c7bc5bef5b8"
15:10:08.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 15:10:06 on dist-test-slave-kz7d"
15:10:08.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.285673 17983 fs_report.cc:383] FS layout report
15:10:08.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:08.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal
15:10:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal
15:10:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data/data
15:10:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:08.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.299053 17983 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:10:08.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.307017 17983 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:08.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.307305 17983 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:08.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.329593 17983 rpc_server.cc:307] RPC server started. Bound to: 127.17.38.189:33571
15:10:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.329612 18050 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.38.189:33571 every 8 connection(s)
15:10:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.331059 17983 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/data/info.pb
15:10:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.337342 17562 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu as pid 17983
15:10:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.337358 18051 sys_catalog.cc:263] Verifying existing consensus state
15:10:08.337 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 15:10:08.337862 18051 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:10:08.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.338112 17983 master.cc:537] Master@127.17.38.189:33571 shutting down...
15:10:08.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:08.351223 17983 master.cc:552] Master@127.17.38.189:33571 shutdown complete.
15:10:08.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskY4X0Fv/test-tmp/mini-kudu-cluster6566313336651774390/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:10:08.397 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.38.188:41141(127.17.38.188:41141)] connecting to peer
15:10:08.473 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:08.475 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:08.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.508314 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:08.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:08.508525 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:08.536 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.17.38.188:41141(127.17.38.188:41141)] Successfully connected to peer
15:10:08.539 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.542 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.607 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
15:10:08.616 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.17.38.188
15:10:08.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:08 dist-test-slave-kz7d krb5kdc[17573](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521806, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.38.188@KRBTEST.COM
15:10:08.816 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xbb3af864, L:/127.0.0.1:46576 - R:/127.17.38.188:41141] using SASL/GSSAPI
15:10:08.831 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:08.845 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:08.849 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:08.850 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.850 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.851 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.852 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.853 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.853 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.854 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:08.863 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:08.865 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.865 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:08.865 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.868 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.868 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.869 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:08.883 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:08.884 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:08.888 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.890 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.890 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.890 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.890 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.891 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:08.903 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:08.905 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:08.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.907 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.910 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:08.943 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:08.944 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:08.946 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.946 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.947 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:08.948 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.948 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:08.949 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:09.003 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:09.005 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:09.006 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:09.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:09.007 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:09.008 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:09.008 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:09.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:09.009 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:09.123 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:09.126 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:09.127 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:09.127 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:09.128 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:09.128 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:09.128 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:09.128 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:09.129 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:09.236255 18058 raft_consensus.cc:482] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:09.236449 18058 raft_consensus.cc:504] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } } peers { permanent_uuid: "75c721a7e130478dae871c7bc5bef5b8" member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } } peers { permanent_uuid: "06f2b15c049a444cb78ebe625650a148" member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } }
15:10:09.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:09.237099 18058 leader_election.cc:288] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925), 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571)
15:10:09.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:09.239496 17844 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925): Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111)
15:10:09.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:09.239641 17845 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571): Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111)
15:10:09.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:09.239758 17845 leader_election.cc:302] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 06f2b15c049a444cb78ebe625650a148; no voters: 63c7afaba27f4ab083c3d882f8cd3e49, 75c721a7e130478dae871c7bc5bef5b8
15:10:09.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:09.240052 18058 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:10:09.383 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:09.384 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:09.386 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:09.386 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:09.387 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:09.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:09.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:09.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:09.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:09.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:09.509764 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:09.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:09.509894 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:09.623 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:09.625 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:09.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:09.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:09.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:09.627 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:09.627 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:09.628 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:09.628 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:10.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:10.511037 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:10.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:10.511166 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:10.623 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:10.625 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:10.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:10.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:10.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:10.627 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:10.627 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:10.627 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:10.628 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:11.512282 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:11.512411 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:11.603 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:11.604 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:11.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:11.606 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:11.607 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:11.608 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:11.608 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:11.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:11.609 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:12.162531 18063 raft_consensus.cc:482] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:12.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:12.162715 18063 raft_consensus.cc:504] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } } peers { permanent_uuid: "75c721a7e130478dae871c7bc5bef5b8" member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } } peers { permanent_uuid: "06f2b15c049a444cb78ebe625650a148" member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } }
15:10:12.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:12.163400 18063 leader_election.cc:288] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925), 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571)
15:10:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:12.165516 17844 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925): Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111)
15:10:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:12.165683 17845 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571): Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111)
15:10:12.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:12.165836 17845 leader_election.cc:302] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 06f2b15c049a444cb78ebe625650a148; no voters: 63c7afaba27f4ab083c3d882f8cd3e49, 75c721a7e130478dae871c7bc5bef5b8
15:10:12.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:12.166052 18063 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:10:12.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:12.513559 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:12.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:12.513700 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:12.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:12.518115 17828 master_runner.cc:186] Error getting master registration for 127.17.38.190:34925: Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111),
15:10:12.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:12.518272 17828 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:13.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:13.514863 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:13.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:13.514981 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:14.443 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:14.444 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:14.447 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:14.447 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:14.447 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:14.447 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:14.447 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:14.448 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:14.448 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:14.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:14.516147 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:14.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:14.516264 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:15.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:15.517472 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:15.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:15.517596 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:16.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:16.344779 18067 raft_consensus.cc:482] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:16.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:16.344967 18067 raft_consensus.cc:504] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } } peers { permanent_uuid: "75c721a7e130478dae871c7bc5bef5b8" member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } } peers { permanent_uuid: "06f2b15c049a444cb78ebe625650a148" member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } }
15:10:16.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:16.345681 18067 leader_election.cc:288] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925), 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571)
15:10:16.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:16.346640 17844 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111) [suppressed 5 similar messages]
15:10:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:16.347919 17844 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925): Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111)
15:10:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:16.348099 17845 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571): Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111)
15:10:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:16.348202 17845 leader_election.cc:302] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 06f2b15c049a444cb78ebe625650a148; no voters: 63c7afaba27f4ab083c3d882f8cd3e49, 75c721a7e130478dae871c7bc5bef5b8
15:10:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:16.348452 18067 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:10:16.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:16.518757 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:16.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:16.518875 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:17.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:17 dist-test-slave-kz7d krb5kdc[17573](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521817, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:17.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:17.405002 17839 init.cc:295] Successfully reacquired a new kerberos TGT
15:10:17.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:17.519706 17828 master_runner.cc:186] Error getting master registration for 127.17.38.190:34925: Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111),
15:10:17.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:17.519899 17828 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:17.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:17.520210 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:17.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:17.520298 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:18.063 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:18.064 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:18.064 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:18.065 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:18.065 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:18.066 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:18.066 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:18.067 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:18.067 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:18.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:18.521478 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:18.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:18.521600 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:19.404 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:19.404 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:19.406 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:19.406 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:19.407 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:19.407 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:19.407 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:19.407 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:19.407 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:19.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:19.522912 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:19.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:19.523061 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:20.524153 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:20.524274 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:21.525382 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:21.525499 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:22.183 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:22.185 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:22.185 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:22.186 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:22.186 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:22.187 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:22.187 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:22.187 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:22.187 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:22.521478 17828 master_runner.cc:186] Error getting master registration for 127.17.38.190:34925: Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111),
15:10:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:22.521672 17828 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:22.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:22.526557 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:22.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:22.526649 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:23.076722 18074 raft_consensus.cc:482] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:23.076923 18074 raft_consensus.cc:504] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } } peers { permanent_uuid: "75c721a7e130478dae871c7bc5bef5b8" member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } } peers { permanent_uuid: "06f2b15c049a444cb78ebe625650a148" member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } }
15:10:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:23.077600 18074 leader_election.cc:288] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925), 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571)
15:10:23.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:23.078684 17845 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111) [suppressed 1 similar messages]
15:10:23.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:23.079998 17844 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925): Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111)
15:10:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:23.080346 17845 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571): Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111)
15:10:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:23.080453 17845 leader_election.cc:302] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 06f2b15c049a444cb78ebe625650a148; no voters: 63c7afaba27f4ab083c3d882f8cd3e49, 75c721a7e130478dae871c7bc5bef5b8
15:10:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:23.080682 18074 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:10:23.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:23.527784 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:23.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:23.527905 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:24.529153 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:24.529335 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:25.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:25.530488 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:25.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:25.530612 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:25.623 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:25.625 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:25.626 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:25.626 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:25.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:25.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:25.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:25.632 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:25.633 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:26.531836 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:26.531952 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:26.969889 18078 raft_consensus.cc:482] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:26.970053 18078 raft_consensus.cc:504] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } } peers { permanent_uuid: "75c721a7e130478dae871c7bc5bef5b8" member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } } peers { permanent_uuid: "06f2b15c049a444cb78ebe625650a148" member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } }
15:10:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:26.970604 18078 leader_election.cc:288] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925), 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571)
15:10:26.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:26.972527 17844 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925): Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111)
15:10:26.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:26.972716 17845 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571): Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111)
15:10:26.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:26.972839 17845 leader_election.cc:302] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 06f2b15c049a444cb78ebe625650a148; no voters: 63c7afaba27f4ab083c3d882f8cd3e49, 75c721a7e130478dae871c7bc5bef5b8
15:10:26.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:26.973088 18078 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:10:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:27 dist-test-slave-kz7d krb5kdc[17573](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521827, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:27.406628 17839 init.cc:295] Successfully reacquired a new kerberos TGT
15:10:27.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:27.522926 17828 master_runner.cc:186] Error getting master registration for 127.17.38.190:34925: Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111),
15:10:27.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:27.523106 17828 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:27.533107 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:27.533232 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:28.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:28.534655 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:28.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:28.534780 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:29.423 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:29.425 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:29.425 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:29.425 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:29.426 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:29.428 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:29.428 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:29.428 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:29.428 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:29.503 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:29.505 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:29.505 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:29.505 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:29.506 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:29.506 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:29.507 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:29.507 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:29.507 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:29.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:29.535988 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:29.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:29.536111 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:30.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:30.537230 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:30.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:30.537348 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:31.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:31.538451 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:31.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:31.538590 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:31.723 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:31.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:31.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:31.728 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:31.731 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:31.734 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:31.735 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:31.735 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:31.735 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:32.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:32.524399 17828 master_runner.cc:186] Error getting master registration for 127.17.38.190:34925: Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111),
15:10:32.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:32.524570 17828 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:32.539609 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:32.539716 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:32.663 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:32.664 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:32.664 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:32.664 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:32.664 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:32.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:32.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:32.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:32.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:33.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:33.134495 18084 raft_consensus.cc:482] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:33.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:33.134667 18084 raft_consensus.cc:504] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "63c7afaba27f4ab083c3d882f8cd3e49" member_type: VOTER last_known_addr { host: "127.17.38.190" port: 34925 } } peers { permanent_uuid: "75c721a7e130478dae871c7bc5bef5b8" member_type: VOTER last_known_addr { host: "127.17.38.189" port: 33571 } } peers { permanent_uuid: "06f2b15c049a444cb78ebe625650a148" member_type: VOTER last_known_addr { host: "127.17.38.188" port: 41141 } }
15:10:33.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:33.135215 18084 leader_election.cc:288] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925), 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571)
15:10:33.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:33.136278 17845 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111) [suppressed 3 similar messages]
15:10:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:33.137398 17845 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 75c721a7e130478dae871c7bc5bef5b8 (127.17.38.189:33571): Network error: Client connection negotiation failed: client connection to 127.17.38.189:33571: connect: Connection refused (error 111)
15:10:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:33.137564 17844 leader_election.cc:334] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 63c7afaba27f4ab083c3d882f8cd3e49 (127.17.38.190:34925): Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111)
15:10:33.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:33.137683 17844 leader_election.cc:302] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 06f2b15c049a444cb78ebe625650a148; no voters: 63c7afaba27f4ab083c3d882f8cd3e49, 75c721a7e130478dae871c7bc5bef5b8
15:10:33.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:33.137890 18084 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:10:33.263 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:33.263 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:33.264 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:33.264 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:33.264 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:33.264 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:33.264 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:33.265 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:33.266 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:33.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:33.540838 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:33.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:33.540984 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:34.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:34.542290 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:34.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:34.542413 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:34.663 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:34.664 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:34.664 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:34.665 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:34.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:34.665 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:34.665 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:34.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:34.666 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:35.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:35.543570 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:35.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:35.543692 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:36.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:36.544868 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:36.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:36.544986 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:37.283 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:37.284 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:37.284 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:37.284 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:37.284 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:37.286 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:37.286 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:37.286 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:37.286 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 15:10:37 dist-test-slave-kz7d krb5kdc[17573](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790521837, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.38.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:37.408293 17839 init.cc:295] Successfully reacquired a new kerberos TGT
15:10:37.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:37.525851 17828 master_runner.cc:186] Error getting master registration for 127.17.38.190:34925: Network error: Client connection negotiation failed: client connection to 127.17.38.190:34925: connect: Connection refused (error 111),
15:10:37.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:37.526034 17828 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:37.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:37.546178 17910 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06f2b15c049a444cb78ebe625650a148: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:37.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:37.546285 17910 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:37.563 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.190:34925(127.17.38.190:34925)] connecting to peer
15:10:37.563 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.38.189:33571(127.17.38.189:33571)] connecting to peer
15:10:37.564 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:37.564 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.38.190:34925(127.17.38.190:34925)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:37.565 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.190:34925: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925
15:10:37.565 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:37.566 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.189:33571(127.17.38.189:33571)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:37.566 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.38.189:33571: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
15:10:37.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571; will retry
15:10:37.571 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29225), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [195ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [245ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [482ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [485ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [496ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [504ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [507ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [508ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [519ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [521ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [524ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [543ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [543ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [544ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [545ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [556ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [559ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [560ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [564ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [602ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [603ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [656ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [660ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [660ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [662ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [663ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [776ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [780ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [782ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [783ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1036ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [1043ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1044ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1276ms] refreshing cache from master, [1276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [1279ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1283ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [2277ms] refreshing cache from master, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [2278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [2280ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [2281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [2281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [2282ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [3256ms] refreshing cache from master, [3257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [3257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [3258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [3260ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [3262ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [3262ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [3263ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [6097ms] refreshing cache from master, [6097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [6097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [6098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [6101ms] too many traces: truncated at 100 traces, deferred=Deferred@503718895(state=PENDING, result=null, callback=(continuation of Deferred@621456002 after retry RPC after error@1162196038) -> (continuation of Deferred@2057586084 after retry RPC after error@1435637828) -> (continuation of Deferred@624748238 after retry RPC after error@1303609771) -> (continuation of Deferred@367158492 after retry RPC after error@1358372380) -> (continuation of Deferred@306157658 after retry RPC after error@1129381231) -> (continuation of Deferred@1158849885 after retry RPC after error@763188253) -> (continuation of Deferred@1131340960 after retry RPC after error@126992935) -> (continuation of Deferred@1467590597 after retry RPC after error@274975883) -> (continuation of Deferred@614232515 after retry RPC after error@756354957) -> (continuation of Deferred@935903845 after retry RPC after error@960579459) -> (continuation of Deferred@2123643649 after retry RPC after error@2071472697) -> (continuation of Deferred@173151449 after retry RPC after error@958146383) -> (continuation of Deferred@1712279390 after retry RPC after error@1553766468) -> (continuation of Deferred@1181924306 after retry RPC after error@504154636) -> (continuation of Deferred@48562988 after retry RPC after error@973337996) -> (continuation of Deferred@1769764149 after retry RPC after error@291020476) -> (continuation of Deferred@990778328 after retry RPC after error@511979168) -> (continuation of Deferred@341517720 after retry RPC after error@1101264081) -> (continuation of Deferred@175685614 after retry RPC after error@901144879) -> (continuation of Deferred@2014115300 after retry RPC after error@643273544) -> (continuation of Deferred@1763692451 after retry RPC after error@1864787126) -> (continuation of Deferred@1880756067 after retry RPC after error@1791827963) -> (continuation of Deferred@2100864982 after retry RPC after error@568815379) -> (continuation of Deferred@19275099 after retry RPC after error@51836995), errback=(continuation of Deferred@621456002 after retry RPC after error@1162196038) -> (continuation of Deferred@2057586084 after retry RPC after error@1435637828) -> (continuation of Deferred@624748238 after retry RPC after error@1303609771) -> (continuation of Deferred@367158492 after retry RPC after error@1358372380) -> (continuation of Deferred@306157658 after retry RPC after error@1129381231) -> (continuation of Deferred@1158849885 after retry RPC after error@763188253) -> (continuation of Deferred@1131340960 after retry RPC after error@126992935) -> (continuation of Deferred@1467590597 after retry RPC after error@274975883) -> (continuation of Deferred@614232515 after retry RPC after error@756354957) -> (continuation of Deferred@935903845 after retry RPC after error@960579459) -> (continuation of Deferred@2123643649 after retry RPC after error@2071472697) -> (continuation of Deferred@173151449 after retry RPC after error@958146383) -> (continuation of Deferred@1712279390 after retry RPC after error@1553766468) -> (continuation of Deferred@1181924306 after retry RPC after error@504154636) -> (continuation of Deferred@48562988 after retry RPC after error@973337996) -> (continuation of Deferred@1769764149 after retry RPC after error@291020476) -> (continuation of Deferred@990778328 after retry RPC after error@511979168) -> (continuation of Deferred@341517720 after retry RPC after error@1101264081) -> (continuation of Deferred@175685614 after retry RPC after error@901144879) -> (continuation of Deferred@2014115300 after retry RPC after error@643273544) -> (continuation of Deferred@1763692451 after retry RPC after error@1864787126) -> (continuation of Deferred@1880756067 after retry RPC after error@1791827963) -> (continuation of Deferred@2100864982 after retry RPC after error@568815379) -> (continuation of Deferred@19275099 after retry RPC after error@51836995)))
15:10:37.576 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.17.38.188:41141(127.17.38.188:41141)] disconnecting while in state READY
15:10:37.577 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.17.38.189:33571(127.17.38.189:33571)] disconnecting while in state TERMINATED
15:10:37.577 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.17.38.190:34925(127.17.38.190:34925)] disconnecting while in state TERMINATED
15:10:37.581 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:10:37.583 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:10:37.584 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.17.38.188:41141(127.17.38.188:41141)] handling channelInactive
15:10:37.584 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.38.188:41141(127.17.38.188:41141)] cleaning up while in state READY due to: connection closed
15:10:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:37.587627 17562 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu pid 17828
15:10:37.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 15:10:37.591586 17562 external_mini_cluster.cc:1652] Unable to check leaks on /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu pid 17828: Not authorized: Client connection negotiation failed: client connection to 127.17.38.188:41141: server requires authentication, but client does not have Kerberos credentials available
15:10:37.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:37.591789 17562 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu with pid 17828
15:10:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T15:10:37Z chronyd exiting
15:10:37.718 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:10:37.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:37.719422 16877 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu pid 17156
15:10:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:37.728302 17278 generic_service.cc:196] Checking for leaks (request via RPC)
15:10:37.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:37.810842 16877 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu with pid 17156
15:10:37.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:37.824004 16877 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu pid 17292
15:10:37.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:37.832209 17415 generic_service.cc:196] Checking for leaks (request via RPC)
15:10:37.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:37.922303 16877 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu with pid 17292
15:10:37.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:37.937063 16877 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu pid 17427
15:10:37.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:37.945560 17549 generic_service.cc:196] Checking for leaks (request via RPC)
15:10:38.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:38.029671 16877 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu with pid 17427
15:10:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:38.042287 16877 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu pid 16888
15:10:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:38.043423 16949 generic_service.cc:196] Checking for leaks (request via RPC)
15:10:38.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:38.149796 16877 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu with pid 16888
15:10:38.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:38.165866 16877 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu pid 16956
15:10:38.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:38.166826 17020 generic_service.cc:196] Checking for leaks (request via RPC)
15:10:38.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:38.269920 16877 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu with pid 16956
15:10:38.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:38.286468 16877 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu pid 17027
15:10:38.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:38.287592 17091 generic_service.cc:196] Checking for leaks (request via RPC)
15:10:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 15:10:38.392889 16877 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskY4X0Fv/build/asan/bin/kudu with pid 17027
15:10:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T15:10:38Z chronyd exiting
15:10:38.528 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29225), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [195ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [245ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [482ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [485ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [496ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [504ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [507ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [508ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [519ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [521ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [524ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [543ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [543ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [544ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [545ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [556ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [559ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [560ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [564ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [602ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [603ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [656ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [660ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [660ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [662ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [663ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [776ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [780ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [782ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [783ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1036ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [1043ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1044ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1276ms] refreshing cache from master, [1276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [1279ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1283ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [2277ms] refreshing cache from master, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [2278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [2280ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [2281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [2281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [2282ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [3256ms] refreshing cache from master, [3257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [3257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [3258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [3260ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [3262ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [3262ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [3263ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [6097ms] refreshing cache from master, [6097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [6097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [6098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [6101ms] too many traces: truncated at 100 traces, deferred=Deferred@503718895(state=PENDING, result=null, callback=(continuation of Deferred@621456002 after retry RPC after error@1162196038) -> (continuation of Deferred@2057586084 after retry RPC after error@1435637828) -> (continuation of Deferred@624748238 after retry RPC after error@1303609771) -> (continuation of Deferred@367158492 after retry RPC after error@1358372380) -> (continuation of Deferred@306157658 after retry RPC after error@1129381231) -> (continuation of Deferred@1158849885 after retry RPC after error@763188253) -> (continuation of Deferred@1131340960 after retry RPC after error@126992935) -> (continuation of Deferred@1467590597 after retry RPC after error@274975883) -> (continuation of Deferred@614232515 after retry RPC after error@756354957) -> (continuation of Deferred@935903845 after retry RPC after error@960579459) -> (continuation of Deferred@2123643649 after retry RPC after error@2071472697) -> (continuation of Deferred@173151449 after retry RPC after error@958146383) -> (continuation of Deferred@1712279390 after retry RPC after error@1553766468) -> (continuation of Deferred@1181924306 after retry RPC after error@504154636) -> (continuation of Deferred@48562988 after retry RPC after error@973337996) -> (continuation of Deferred@1769764149 after retry RPC after error@291020476) -> (continuation of Deferred@990778328 after retry RPC after error@511979168) -> (continuation of Deferred@341517720 after retry RPC after error@1101264081) -> (continuation of Deferred@175685614 after retry RPC after error@901144879) -> (continuation of Deferred@2014115300 after retry RPC after error@643273544) -> (continuation of Deferred@1763692451 after retry RPC after error@1864787126) -> (continuation of Deferred@1880756067 after retry RPC after error@1791827963) -> (continuation of Deferred@2100864982 after retry RPC after error@568815379) -> (continuation of Deferred@19275099 after retry RPC after error@51836995), errback=(continuation of Deferred@621456002 after retry RPC after error@1162196038) -> (continuation of Deferred@2057586084 after retry RPC after error@1435637828) -> (continuation of Deferred@624748238 after retry RPC after error@1303609771) -> (continuation of Deferred@367158492 after retry RPC after error@1358372380) -> (continuation of Deferred@306157658 after retry RPC after error@1129381231) -> (continuation of Deferred@1158849885 after retry RPC after error@763188253) -> (continuation of Deferred@1131340960 after retry RPC after error@126992935) -> (continuation of Deferred@1467590597 after retry RPC after error@274975883) -> (continuation of Deferred@614232515 after retry RPC after error@756354957) -> (continuation of Deferred@935903845 after retry RPC after error@960579459) -> (continuation of Deferred@2123643649 after retry RPC after error@2071472697) -> (continuation of Deferred@173151449 after retry RPC after error@958146383) -> (continuation of Deferred@1712279390 after retry RPC after error@1553766468) -> (continuation of Deferred@1181924306 after retry RPC after error@504154636) -> (continuation of Deferred@48562988 after retry RPC after error@973337996) -> (continuation of Deferred@1769764149 after retry RPC after error@291020476) -> (continuation of Deferred@990778328 after retry RPC after error@511979168) -> (continuation of Deferred@341517720 after retry RPC after error@1101264081) -> (continuation of Deferred@175685614 after retry RPC after error@901144879) -> (continuation of Deferred@2014115300 after retry RPC after error@643273544) -> (continuation of Deferred@1763692451 after retry RPC after error@1864787126) -> (continuation of Deferred@1880756067 after retry RPC after error@1791827963) -> (continuation of Deferred@2100864982 after retry RPC after error@568815379) -> (continuation of Deferred@19275099 after retry RPC after error@51836995)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.17.38.189:33571
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29225), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [195ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [245ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [482ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [485ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [496ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [504ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [507ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [508ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [519ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [521ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [524ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [543ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [543ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [544ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [545ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [556ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [559ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [560ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [564ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [602ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [603ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [656ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [660ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [660ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [662ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [663ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [776ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [780ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [782ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [783ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1036ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [1043ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1044ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1276ms] refreshing cache from master, [1276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [1279ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1283ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [2277ms] refreshing cache from master, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [2278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [2280ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [2281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [2281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [2282ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [3256ms] refreshing cache from master, [3257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [3257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [3258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [3260ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [3262ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [3262ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [3263ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [6097ms] refreshing cache from master, [6097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [6097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [6098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [6101ms] too many traces: truncated at 100 traces, deferred=Deferred@503718895(state=PENDING, result=null, callback=(continuation of Deferred@621456002 after retry RPC after error@1162196038) -> (continuation of Deferred@2057586084 after retry RPC after error@1435637828) -> (continuation of Deferred@624748238 after retry RPC after error@1303609771) -> (continuation of Deferred@367158492 after retry RPC after error@1358372380) -> (continuation of Deferred@306157658 after retry RPC after error@1129381231) -> (continuation of Deferred@1158849885 after retry RPC after error@763188253) -> (continuation of Deferred@1131340960 after retry RPC after error@126992935) -> (continuation of Deferred@1467590597 after retry RPC after error@274975883) -> (continuation of Deferred@614232515 after retry RPC after error@756354957) -> (continuation of Deferred@935903845 after retry RPC after error@960579459) -> (continuation of Deferred@2123643649 after retry RPC after error@2071472697) -> (continuation of Deferred@173151449 after retry RPC after error@958146383) -> (continuation of Deferred@1712279390 after retry RPC after error@1553766468) -> (continuation of Deferred@1181924306 after retry RPC after error@504154636) -> (continuation of Deferred@48562988 after retry RPC after error@973337996) -> (continuation of Deferred@1769764149 after retry RPC after error@291020476) -> (continuation of Deferred@990778328 after retry RPC after error@511979168) -> (continuation of Deferred@341517720 after retry RPC after error@1101264081) -> (continuation of Deferred@175685614 after retry RPC after error@901144879) -> (continuation of Deferred@2014115300 after retry RPC after error@643273544) -> (continuation of Deferred@1763692451 after retry RPC after error@1864787126) -> (continuation of Deferred@1880756067 after retry RPC after error@1791827963) -> (continuation of Deferred@2100864982 after retry RPC after error@568815379) -> (continuation of Deferred@19275099 after retry RPC after error@51836995), errback=(continuation of Deferred@621456002 after retry RPC after error@1162196038) -> (continuation of Deferred@2057586084 after retry RPC after error@1435637828) -> (continuation of Deferred@624748238 after retry RPC after error@1303609771) -> (continuation of Deferred@367158492 after retry RPC after error@1358372380) -> (continuation of Deferred@306157658 after retry RPC after error@1129381231) -> (continuation of Deferred@1158849885 after retry RPC after error@763188253) -> (continuation of Deferred@1131340960 after retry RPC after error@126992935) -> (continuation of Deferred@1467590597 after retry RPC after error@274975883) -> (continuation of Deferred@614232515 after retry RPC after error@756354957) -> (continuation of Deferred@935903845 after retry RPC after error@960579459) -> (continuation of Deferred@2123643649 after retry RPC after error@2071472697) -> (continuation of Deferred@173151449 after retry RPC after error@958146383) -> (continuation of Deferred@1712279390 after retry RPC after error@1553766468) -> (continuation of Deferred@1181924306 after retry RPC after error@504154636) -> (continuation of Deferred@48562988 after retry RPC after error@973337996) -> (continuation of Deferred@1769764149 after retry RPC after error@291020476) -> (continuation of Deferred@990778328 after retry RPC after error@511979168) -> (continuation of Deferred@341517720 after retry RPC after error@1101264081) -> (continuation of Deferred@175685614 after retry RPC after error@901144879) -> (continuation of Deferred@2014115300 after retry RPC after error@643273544) -> (continuation of Deferred@1763692451 after retry RPC after error@1864787126) -> (continuation of Deferred@1880756067 after retry RPC after error@1791827963) -> (continuation of Deferred@2100864982 after retry RPC after error@568815379) -> (continuation of Deferred@19275099 after retry RPC after error@51836995)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
... 14 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.17.38.189:33571
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
15:10:38.878 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
15:10:38.878 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29225), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [195ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [245ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [482ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [485ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [496ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [504ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [507ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [508ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [519ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [521ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [524ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [543ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [543ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [544ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [545ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [556ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [559ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [560ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [564ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [602ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [603ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [656ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [660ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [660ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [662ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [663ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [776ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [780ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [782ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [783ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1036ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [1043ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1044ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1276ms] refreshing cache from master, [1276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [1279ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1283ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [2277ms] refreshing cache from master, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [2278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [2280ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [2281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [2281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [2282ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [3256ms] refreshing cache from master, [3257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [3257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [3258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [3260ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [3262ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [3262ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [3263ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [6097ms] refreshing cache from master, [6097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [6097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [6098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [6101ms] too many traces: truncated at 100 traces, deferred=Deferred@503718895(state=PENDING, result=null, callback=(continuation of Deferred@621456002 after retry RPC after error@1162196038) -> (continuation of Deferred@2057586084 after retry RPC after error@1435637828) -> (continuation of Deferred@624748238 after retry RPC after error@1303609771) -> (continuation of Deferred@367158492 after retry RPC after error@1358372380) -> (continuation of Deferred@306157658 after retry RPC after error@1129381231) -> (continuation of Deferred@1158849885 after retry RPC after error@763188253) -> (continuation of Deferred@1131340960 after retry RPC after error@126992935) -> (continuation of Deferred@1467590597 after retry RPC after error@274975883) -> (continuation of Deferred@614232515 after retry RPC after error@756354957) -> (continuation of Deferred@935903845 after retry RPC after error@960579459) -> (continuation of Deferred@2123643649 after retry RPC after error@2071472697) -> (continuation of Deferred@173151449 after retry RPC after error@958146383) -> (continuation of Deferred@1712279390 after retry RPC after error@1553766468) -> (continuation of Deferred@1181924306 after retry RPC after error@504154636) -> (continuation of Deferred@48562988 after retry RPC after error@973337996) -> (continuation of Deferred@1769764149 after retry RPC after error@291020476) -> (continuation of Deferred@990778328 after retry RPC after error@511979168) -> (continuation of Deferred@341517720 after retry RPC after error@1101264081) -> (continuation of Deferred@175685614 after retry RPC after error@901144879) -> (continuation of Deferred@2014115300 after retry RPC after error@643273544) -> (continuation of Deferred@1763692451 after retry RPC after error@1864787126) -> (continuation of Deferred@1880756067 after retry RPC after error@1791827963) -> (continuation of Deferred@2100864982 after retry RPC after error@568815379) -> (continuation of Deferred@19275099 after retry RPC after error@51836995), errback=(continuation of Deferred@621456002 after retry RPC after error@1162196038) -> (continuation of Deferred@2057586084 after retry RPC after error@1435637828) -> (continuation of Deferred@624748238 after retry RPC after error@1303609771) -> (continuation of Deferred@367158492 after retry RPC after error@1358372380) -> (continuation of Deferred@306157658 after retry RPC after error@1129381231) -> (continuation of Deferred@1158849885 after retry RPC after error@763188253) -> (continuation of Deferred@1131340960 after retry RPC after error@126992935) -> (continuation of Deferred@1467590597 after retry RPC after error@274975883) -> (continuation of Deferred@614232515 after retry RPC after error@756354957) -> (continuation of Deferred@935903845 after retry RPC after error@960579459) -> (continuation of Deferred@2123643649 after retry RPC after error@2071472697) -> (continuation of Deferred@173151449 after retry RPC after error@958146383) -> (continuation of Deferred@1712279390 after retry RPC after error@1553766468) -> (continuation of Deferred@1181924306 after retry RPC after error@504154636) -> (continuation of Deferred@48562988 after retry RPC after error@973337996) -> (continuation of Deferred@1769764149 after retry RPC after error@291020476) -> (continuation of Deferred@990778328 after retry RPC after error@511979168) -> (continuation of Deferred@341517720 after retry RPC after error@1101264081) -> (continuation of Deferred@175685614 after retry RPC after error@901144879) -> (continuation of Deferred@2014115300 after retry RPC after error@643273544) -> (continuation of Deferred@1763692451 after retry RPC after error@1864787126) -> (continuation of Deferred@1880756067 after retry RPC after error@1791827963) -> (continuation of Deferred@2100864982 after retry RPC after error@568815379) -> (continuation of Deferred@19275099 after retry RPC after error@51836995)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.17.38.189:33571
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29225), Traces: [0ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [195ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [245ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [482ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [485ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [496ms] refreshing cache from master, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [499ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [502ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [504ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [507ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [508ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [519ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [519ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [521ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [524ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [543ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [543ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [544ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [545ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [556ms] refreshing cache from master, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [559ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [560ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [563ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [564ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [601ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [602ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [603ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [656ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [660ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [660ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [662ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [663ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [776ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [780ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [781ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [782ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [783ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1036ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [1038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [1040ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [1043ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1044ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1276ms] refreshing cache from master, [1276ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [1279ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [1281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [1283ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [2277ms] refreshing cache from master, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [2277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [2278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [2280ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [2281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [2281ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [2282ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [3256ms] refreshing cache from master, [3257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [3257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [3258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [3260ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.190:34925: Network error: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925, [3262ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.188:41141: OK, [3262ms] Sub RPC ConnectToMaster: received response from server master-127.17.38.189:33571: Network error: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [3263ms] delaying RPC due to: Service unavailable: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571, [6097ms] refreshing cache from master, [6097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.188:41141, [6097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.190:34925, [6098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.38.189:33571, [6101ms] too many traces: truncated at 100 traces, deferred=Deferred@503718895(state=PENDING, result=null, callback=(continuation of Deferred@621456002 after retry RPC after error@1162196038) -> (continuation of Deferred@2057586084 after retry RPC after error@1435637828) -> (continuation of Deferred@624748238 after retry RPC after error@1303609771) -> (continuation of Deferred@367158492 after retry RPC after error@1358372380) -> (continuation of Deferred@306157658 after retry RPC after error@1129381231) -> (continuation of Deferred@1158849885 after retry RPC after error@763188253) -> (continuation of Deferred@1131340960 after retry RPC after error@126992935) -> (continuation of Deferred@1467590597 after retry RPC after error@274975883) -> (continuation of Deferred@614232515 after retry RPC after error@756354957) -> (continuation of Deferred@935903845 after retry RPC after error@960579459) -> (continuation of Deferred@2123643649 after retry RPC after error@2071472697) -> (continuation of Deferred@173151449 after retry RPC after error@958146383) -> (continuation of Deferred@1712279390 after retry RPC after error@1553766468) -> (continuation of Deferred@1181924306 after retry RPC after error@504154636) -> (continuation of Deferred@48562988 after retry RPC after error@973337996) -> (continuation of Deferred@1769764149 after retry RPC after error@291020476) -> (continuation of Deferred@990778328 after retry RPC after error@511979168) -> (continuation of Deferred@341517720 after retry RPC after error@1101264081) -> (continuation of Deferred@175685614 after retry RPC after error@901144879) -> (continuation of Deferred@2014115300 after retry RPC after error@643273544) -> (continuation of Deferred@1763692451 after retry RPC after error@1864787126) -> (continuation of Deferred@1880756067 after retry RPC after error@1791827963) -> (continuation of Deferred@2100864982 after retry RPC after error@568815379) -> (continuation of Deferred@19275099 after retry RPC after error@51836995), errback=(continuation of Deferred@621456002 after retry RPC after error@1162196038) -> (continuation of Deferred@2057586084 after retry RPC after error@1435637828) -> (continuation of Deferred@624748238 after retry RPC after error@1303609771) -> (continuation of Deferred@367158492 after retry RPC after error@1358372380) -> (continuation of Deferred@306157658 after retry RPC after error@1129381231) -> (continuation of Deferred@1158849885 after retry RPC after error@763188253) -> (continuation of Deferred@1131340960 after retry RPC after error@126992935) -> (continuation of Deferred@1467590597 after retry RPC after error@274975883) -> (continuation of Deferred@614232515 after retry RPC after error@756354957) -> (continuation of Deferred@935903845 after retry RPC after error@960579459) -> (continuation of Deferred@2123643649 after retry RPC after error@2071472697) -> (continuation of Deferred@173151449 after retry RPC after error@958146383) -> (continuation of Deferred@1712279390 after retry RPC after error@1553766468) -> (continuation of Deferred@1181924306 after retry RPC after error@504154636) -> (continuation of Deferred@48562988 after retry RPC after error@973337996) -> (continuation of Deferred@1769764149 after retry RPC after error@291020476) -> (continuation of Deferred@990778328 after retry RPC after error@511979168) -> (continuation of Deferred@341517720 after retry RPC after error@1101264081) -> (continuation of Deferred@175685614 after retry RPC after error@901144879) -> (continuation of Deferred@2014115300 after retry RPC after error@643273544) -> (continuation of Deferred@1763692451 after retry RPC after error@1864787126) -> (continuation of Deferred@1880756067 after retry RPC after error@1791827963) -> (continuation of Deferred@2100864982 after retry RPC after error@568815379) -> (continuation of Deferred@19275099 after retry RPC after error@51836995)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.17.38.188:41141,127.17.38.190:34925,127.17.38.189:33571) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.190:34925(127.17.38.190:34925): Connection refused: /127.17.38.190:34925,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.failOrRetryRpc(RpcProxy.java:475)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:252)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.38.189:33571(127.17.38.189:33571): Connection refused: /127.17.38.189:33571
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
... 14 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.17.38.189:33571
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)