19:33:14.643 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
19:33:14.655 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQtGm0f/test-tmp
19:33:14.657 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
19:33:14.661 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQtGm0f/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
19:33:14.924 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
19:33:15.054 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:33:15.055 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
19:33:15.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:15.058269 2910 mini_oidc.cc:152] Starting JWKS server
19:33:15.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:15.062299 2910 webserver.cc:460] Webserver started at http://127.0.0.1:38749/ using document root <none> and password file <none>
19:33:15.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:15.062750 2910 mini_oidc.cc:189] Starting OIDC Discovery server
19:33:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:15.064462 2910 webserver.cc:460] Webserver started at http://127.0.0.1:46175/ using document root <none> and password file <none>
19:33:15.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T19:33:15Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
19:33:15.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T19:33:15Z Disabled control of system clock
19:33:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:15.114454 2910 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/wal
19:33:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/data
19:33:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:33:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:33:15.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:33:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/logs
19:33:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/data/info.pb
19:33:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:33:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:33:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:33:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:33:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:33:15.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:33:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.215.190:39633
19:33:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.215.190
19:33:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.215.148:34939
19:33:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.215.190:39633,127.2.215.189:34297,127.2.215.188:46807
19:33:15.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:15.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38749/jwks.json with env {}
19:33:15.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:15.407375 2922 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:33:15.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:15.407922 2922 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:33:15.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:15.408593 2922 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:33:15.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:15.440125 2922 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:33:15.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:15.440469 2922 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:33:15.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:15.440748 2922 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:33:15.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:15.441053 2922 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:33:15.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:15.441344 2922 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:33:15.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:15.441576 2922 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38749/jwks.json
19:33:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:15.477226 2922 master_runner.cc:386] Master server non-default flags:
19:33:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:15.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.215.148:34939
19:33:15.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:15.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:15.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/data
19:33:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/wal
19:33:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.215.190:39633,127.2.215.189:34297,127.2.215.188:46807
19:33:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.215.190:39633
19:33:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:33:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38749/jwks.json
19:33:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/data/info.pb
19:33:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.215.190
19:33:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:33:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:33:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:33:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/logs
19:33:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:33:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:33:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:33:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:33:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:33:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:33:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 18:42:06 UTC on ccycloud.quasar-gwconk.root.comops.site
19:33:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673102
19:33:15.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:33:15.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:15.478464 2922 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:33:15.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:15.480770 2922 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:33:15.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:15.499534 2928 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:16.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:15.499676 2929 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:16.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:15.499591 2931 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:16.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:15.501379 2922 server_base.cc:1034] running on GCE node
19:33:16.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.770789 2922 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:33:16.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.773542 2922 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:33:16.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.774899 2922 hybrid_clock.cc:648] HybridClock initialized: now 1790364796774866 us; error 72 us; skew 500 ppm
19:33:16.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.775701 2922 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:33:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.782584 2922 webserver.cc:460] Webserver started at http://127.2.215.190:40001/ using document root <none> and password file <none>
19:33:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.783450 2922 fs_manager.cc:359] Metadata directory not provided
19:33:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.783638 2922 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:33:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.784000 2922 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:33:16.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.788830 2922 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/data/instance:
19:33:16.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eae354d6bcb347a680e647222635cc98"
19:33:16.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:16 on dist-test-slave-555z"
19:33:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.789984 2922 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/wal/instance:
19:33:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eae354d6bcb347a680e647222635cc98"
19:33:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:16 on dist-test-slave-555z"
19:33:16.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.797348 2922 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
19:33:16.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.802677 2938 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:33:16.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.803723 2922 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
19:33:16.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.804056 2922 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/data,/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/wal
19:33:16.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eae354d6bcb347a680e647222635cc98"
19:33:16.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:16 on dist-test-slave-555z"
19:33:16.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.804509 2922 fs_report.cc:383] FS layout report
19:33:16.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:33:16.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/wal
19:33:16.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/wal
19:33:16.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/data/data
19:33:16.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:33:16.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:33:16.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:33:16.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:33:16.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:33:16.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:33:16.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:33:16.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:33:16.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:33:16.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:33:16.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:33:16.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.845808 2922 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:33:16.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.847409 2922 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:33:16.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.847930 2922 kserver.cc:163] Server-wide thread pool size limit: 3276
19:33:16.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.914726 2922 rpc_server.cc:307] RPC server started. Bound to: 127.2.215.190:39633
19:33:16.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.914821 2989 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.215.190:39633 every 8 connection(s)
19:33:16.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.917637 2922 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/data/info.pb
19:33:16.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.921989 2910 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu as pid 2922
19:33:16.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.922518 2910 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-0/wal/instance
19:33:16.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.923568 2990 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:33:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.927623 2910 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/wal
19:33:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/data
19:33:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:33:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:33:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:33:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/logs
19:33:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/data/info.pb
19:33:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:33:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:33:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:33:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:33:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:33:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:33:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.215.189:34297
19:33:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.215.189
19:33:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.215.148:34939
19:33:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.215.190:39633,127.2.215.189:34297,127.2.215.188:46807
19:33:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38749/jwks.json with env {}
19:33:16.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.938676 2990 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } has no permanent_uuid. Determining permanent_uuid...
19:33:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:16.966682 2990 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } has no permanent_uuid. Determining permanent_uuid...
19:33:16.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:16.969172 2940 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.215.189:34297: connect: Connection refused (error 111)
19:33:16.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:16.972924 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.189:34297: Network error: Client connection negotiation failed: client connection to 127.2.215.189:34297: connect: Connection refused (error 111)
19:33:17.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:17.022399 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } attempt: 1
19:33:17.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.026218 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.189:34297: Network error: Client connection negotiation failed: client connection to 127.2.215.189:34297: connect: Connection refused (error 111)
19:33:17.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:17.094702 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } attempt: 2
19:33:17.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.099606 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.189:34297: Network error: Client connection negotiation failed: client connection to 127.2.215.189:34297: connect: Connection refused (error 111)
19:33:17.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:17.191028 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } attempt: 3
19:33:17.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.194636 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.189:34297: Network error: Client connection negotiation failed: client connection to 127.2.215.189:34297: connect: Connection refused (error 111)
19:33:17.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.240773 2992 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:33:17.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.241329 2992 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:33:17.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.241873 2992 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:33:17.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.272989 2992 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:33:17.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.273272 2992 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:33:17.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.273456 2992 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:33:17.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.273633 2992 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:33:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.273813 2992 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:33:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.274060 2992 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38749/jwks.json
19:33:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:17.310021 2992 master_runner.cc:386] Master server non-default flags:
19:33:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.215.148:34939
19:33:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/data
19:33:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/wal
19:33:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.215.190:39633,127.2.215.189:34297,127.2.215.188:46807
19:33:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.215.189:34297
19:33:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:33:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38749/jwks.json
19:33:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/data/info.pb
19:33:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.215.189
19:33:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:33:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:33:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:33:17.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:17.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/logs
19:33:17.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:17.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:33:17.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:33:17.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:33:17.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:33:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:33:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:33:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 18:42:06 UTC on ccycloud.quasar-gwconk.root.comops.site
19:33:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673102
19:33:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:33:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:17.311322 2992 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:33:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:17.313030 2992 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:33:17.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.333050 3004 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:17.338018 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } attempt: 4
19:33:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.342115 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.189:34297: Network error: Client connection negotiation failed: client connection to 127.2.215.189:34297: connect: Connection refused (error 111)
19:33:17.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:17.641602 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } attempt: 5
19:33:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.647099 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.189:34297: Network error: Client connection negotiation failed: client connection to 127.2.215.189:34297: connect: Connection refused (error 111)
19:33:18.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.194561 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } attempt: 6
19:33:18.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:18.202754 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.189:34297: Network error: Client connection negotiation failed: client connection to 127.2.215.189:34297: connect: Connection refused (error 111)
19:33:18.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.333096 3002 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:18.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:17.333408 3001 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:18.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:18.538972 3003 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
19:33:18.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.539081 2992 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:33:18.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.543622 2992 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:33:18.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.546958 2992 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:33:18.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.548359 2992 hybrid_clock.cc:648] HybridClock initialized: now 1790364798548320 us; error 76 us; skew 500 ppm
19:33:18.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.549466 2992 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:33:18.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.556571 2992 webserver.cc:460] Webserver started at http://127.2.215.189:34933/ using document root <none> and password file <none>
19:33:18.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.557432 2992 fs_manager.cc:359] Metadata directory not provided
19:33:18.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.557605 2992 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:33:18.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.557972 2992 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:33:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.562772 2992 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/data/instance:
19:33:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4"
19:33:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:18 on dist-test-slave-555z"
19:33:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.564015 2992 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/wal/instance:
19:33:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4"
19:33:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:18 on dist-test-slave-555z"
19:33:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.571288 2992 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.002s sys 0.004s
19:33:18.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.576943 3013 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:33:18.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.577972 2992 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
19:33:18.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.578342 2992 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/data,/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/wal
19:33:18.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4"
19:33:18.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:18 on dist-test-slave-555z"
19:33:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.578771 2992 fs_report.cc:383] FS layout report
19:33:18.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:33:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/wal
19:33:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/wal
19:33:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/data/data
19:33:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:33:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:33:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:33:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:33:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:33:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:33:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:33:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:33:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:33:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:33:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:33:18.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.635758 2992 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:33:18.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.637712 2992 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:33:18.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.638216 2992 kserver.cc:163] Server-wide thread pool size limit: 3276
19:33:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.715090 2992 rpc_server.cc:307] RPC server started. Bound to: 127.2.215.189:34297
19:33:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.715195 3064 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.215.189:34297 every 8 connection(s)
19:33:18.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.717867 2992 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/data/info.pb
19:33:18.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.721696 2910 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu as pid 2992
19:33:18.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.722177 2910 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-1/wal/instance
19:33:18.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.724396 3065 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.725653 2910 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/wal
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/data
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/logs
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/data/info.pb
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:33:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.215.188:46807
19:33:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.215.188
19:33:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.215.148:34939
19:33:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.215.190:39633,127.2.215.189:34297,127.2.215.188:46807
19:33:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:18.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38749/jwks.json with env {}
19:33:18.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.739001 3065 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } has no permanent_uuid. Determining permanent_uuid...
19:33:18.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.764837 3065 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } has no permanent_uuid. Determining permanent_uuid...
19:33:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.783007 3065 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } has no permanent_uuid. Determining permanent_uuid...
19:33:18.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:18.785442 3017 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:18.790030 3065 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:18.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.839644 3065 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 1
19:33:18.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:18.843550 3065 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:18.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:18.912014 3065 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 2
19:33:18.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:18.916767 3065 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:19.008255 3065 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 3
19:33:19.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.013201 3065 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:19.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.054749 3067 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:33:19.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.055311 3067 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:33:19.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.055974 3067 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:33:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.086614 3067 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:33:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.086894 3067 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:33:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.087078 3067 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:33:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.087261 3067 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:33:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.087546 3067 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:33:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.087777 3067 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38749/jwks.json
19:33:19.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:19.121982 3067 master_runner.cc:386] Master server non-default flags:
19:33:19.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:19.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.215.148:34939
19:33:19.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:19.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:19.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/data
19:33:19.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/wal
19:33:19.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:19.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.215.190:39633,127.2.215.189:34297,127.2.215.188:46807
19:33:19.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:19.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.215.188:46807
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38749/jwks.json
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/data/info.pb
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.215.188
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:33:19.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:33:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/logs
19:33:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:33:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:33:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:33:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:33:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:33:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:33:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 18:42:06 UTC on ccycloud.quasar-gwconk.root.comops.site
19:33:19.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673102
19:33:19.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:33:19.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:19.123202 3067 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:33:19.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:19.125277 3067 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:33:19.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.146494 3078 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:19.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:19.156670 3065 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 4
19:33:19.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.160244 3065 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:19.263237 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } attempt: 7
19:33:19.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:19.278734 2990 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } has no permanent_uuid. Determining permanent_uuid...
19:33:19.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.283823 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:19.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:19.342437 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 1
19:33:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.346946 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:19.428555 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 2
19:33:19.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.432957 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:19.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:19.459729 3065 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 5
19:33:19.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.464210 3065 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:19.518535 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 3
19:33:19.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.524459 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:19.664924 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 4
19:33:19.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.670251 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:19.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:19.953632 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 5
19:33:19.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.958338 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:20.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.011709 3065 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 6
19:33:20.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:20.020758 3065 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:20.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.146499 3077 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:20.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:19.146579 3080 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:20.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:20.378386 3079 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
19:33:20.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.378504 3067 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:33:20.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.383281 3067 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:33:20.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.386195 3067 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:33:20.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.387611 3067 hybrid_clock.cc:648] HybridClock initialized: now 1790364800387548 us; error 79 us; skew 500 ppm
19:33:20.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.388607 3067 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:33:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.396194 3067 webserver.cc:460] Webserver started at http://127.2.215.188:36235/ using document root <none> and password file <none>
19:33:20.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.397369 3067 fs_manager.cc:359] Metadata directory not provided
19:33:20.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.397627 3067 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:33:20.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.398208 3067 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:33:20.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.403075 3067 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/data/instance:
19:33:20.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99b8d4b23d8349a2bfd609f2f81e974f"
19:33:20.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:20 on dist-test-slave-555z"
19:33:20.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.404604 3067 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/wal/instance:
19:33:20.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99b8d4b23d8349a2bfd609f2f81e974f"
19:33:20.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:20 on dist-test-slave-555z"
19:33:20.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.412302 3067 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
19:33:20.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.418170 3091 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:33:20.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.419365 3067 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
19:33:20.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.419795 3067 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/data,/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/wal
19:33:20.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99b8d4b23d8349a2bfd609f2f81e974f"
19:33:20.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:20 on dist-test-slave-555z"
19:33:20.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.420346 3067 fs_report.cc:383] FS layout report
19:33:20.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:33:20.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/wal
19:33:20.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/wal
19:33:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/data/data
19:33:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:33:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:33:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:33:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:33:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:33:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:33:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:33:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:33:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:33:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:33:20.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:33:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.467644 3067 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:33:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.469431 3067 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:33:20.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.469874 3067 kserver.cc:163] Server-wide thread pool size limit: 3276
19:33:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.510762 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 6
19:33:20.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:20.518213 2990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.215.188:46807: Network error: Client connection negotiation failed: client connection to 127.2.215.188:46807: connect: Connection refused (error 111)
19:33:20.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.553691 3067 rpc_server.cc:307] RPC server started. Bound to: 127.2.215.188:46807
19:33:20.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.553768 3144 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.215.188:46807 every 8 connection(s)
19:33:20.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.556814 3067 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/data/info.pb
19:33:20.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.557523 2910 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu as pid 3067
19:33:20.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.558035 2910 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/master-2/wal/instance
19:33:20.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.563364 3145 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:33:20.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.582258 3145 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } has no permanent_uuid. Determining permanent_uuid...
19:33:20.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.613408 3145 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } has no permanent_uuid. Determining permanent_uuid...
19:33:20.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.625730 3145 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } has no permanent_uuid. Determining permanent_uuid...
19:33:20.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.645543 3145 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f: Bootstrap starting.
19:33:20.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.652590 3145 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f: Neither blocks nor log segments found. Creating new log.
19:33:20.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.654111 3145 log.cc:824] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f: Log is configured to *not* fsync() on all Append() calls
19:33:20.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.659596 3145 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f: No bootstrap required, opened a new log
19:33:20.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.683979 3145 raft_consensus.cc:348] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } }
19:33:20.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.684829 3145 raft_consensus.cc:374] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:33:20.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.685089 3145 raft_consensus.cc:729] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 99b8d4b23d8349a2bfd609f2f81e974f, State: Initialized, Role: FOLLOWER
19:33:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.685936 3145 consensus_queue.cc:260] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } }
19:33:20.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.688866 3154 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } } }
19:33:20.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.689446 3154 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [sys.catalog]: This master's current role is: FOLLOWER
19:33:20.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.690258 3145 sys_catalog.cc:564] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [sys.catalog]: configured and running, proceeding with master startup.
19:33:20.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:20.725368 3097 tablet.cc:2369] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:33:20.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:20.728726 3165 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:20.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:20.729861 3165 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.713052 3067 master_runner.cc:186] Error getting master registration for 127.2.215.190:39633: OK, instance_id { permanent_uuid: "eae354d6bcb347a680e647222635cc98" instance_seqno: 1790364796896562 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:33:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.747365 3067 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:33:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.866796 3154 raft_consensus.cc:482] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:33:20.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.867316 3154 raft_consensus.cc:504] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } }
19:33:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:20.870375 3154 leader_election.cc:288] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers eae354d6bcb347a680e647222635cc98 (127.2.215.190:39633), 83f8f3ba27ba41ac9fd6ddb3b762e9b4 (127.2.215.189:34297)
19:33:21.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.081152 3065 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 7
19:33:21.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.115482 3065 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4: Bootstrap starting.
19:33:21.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.121496 3065 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4: Neither blocks nor log segments found. Creating new log.
19:33:21.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.125021 3065 log.cc:824] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4: Log is configured to *not* fsync() on all Append() calls
19:33:21.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.130016 3065 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4: No bootstrap required, opened a new log
19:33:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.164345 3065 raft_consensus.cc:348] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } }
19:33:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.165335 3065 raft_consensus.cc:374] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:33:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.166121 3065 raft_consensus.cc:729] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83f8f3ba27ba41ac9fd6ddb3b762e9b4, State: Initialized, Role: FOLLOWER
19:33:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.167145 3065 consensus_queue.cc:260] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } }
19:33:21.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.170001 3169 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } } }
19:33:21.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.170954 3169 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [sys.catalog]: This master's current role is: FOLLOWER
19:33:21.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.172080 3065 sys_catalog.cc:564] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [sys.catalog]: configured and running, proceeding with master startup.
19:33:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.173705 3040 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" is_pre_election: true
19:33:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.174764 3040 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 99b8d4b23d8349a2bfd609f2f81e974f in term 0.
19:33:21.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.176754 3093 leader_election.cc:302] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 83f8f3ba27ba41ac9fd6ddb3b762e9b4, 99b8d4b23d8349a2bfd609f2f81e974f; no voters:
19:33:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.177577 3154 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [term 0 FOLLOWER]: Leader pre-election won for term 1
19:33:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.177958 3154 raft_consensus.cc:482] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
19:33:21.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.178315 3154 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [term 0 FOLLOWER]: Advancing to term 1
19:33:21.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.185689 3154 raft_consensus.cc:504] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } }
19:33:21.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.190059 3154 leader_election.cc:288] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [CANDIDATE]: Term 1 election: Requested vote from peers eae354d6bcb347a680e647222635cc98 (127.2.215.190:39633), 83f8f3ba27ba41ac9fd6ddb3b762e9b4 (127.2.215.189:34297)
19:33:21.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.191161 3040 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4"
19:33:21.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.191835 3040 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [term 0 FOLLOWER]: Advancing to term 1
19:33:21.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.203578 3040 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 99b8d4b23d8349a2bfd609f2f81e974f in term 1.
19:33:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.204797 3093 leader_election.cc:302] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 83f8f3ba27ba41ac9fd6ddb3b762e9b4, 99b8d4b23d8349a2bfd609f2f81e974f; no voters:
19:33:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.205735 3154 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [term 1 FOLLOWER]: Leader election won for term 1
19:33:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.216122 3154 raft_consensus.cc:686] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [term 1 LEADER]: Becoming Leader. State: Replica: 99b8d4b23d8349a2bfd609f2f81e974f, State: Running, Role: LEADER
19:33:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.217733 3154 consensus_queue.cc:237] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } }
19:33:21.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.245971 3180 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 99b8d4b23d8349a2bfd609f2f81e974f. Latest consensus state: current_term: 1 leader_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } } }
19:33:21.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.248051 3180 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [sys.catalog]: This master's current role is: LEADER
19:33:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:21.249545 3181 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:21.249876 3181 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.268975 3184 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
19:33:21.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.288565 3184 catalog_manager.cc:1484] Initializing Kudu cluster ID...
19:33:21.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.228235 2992 master_runner.cc:186] Error getting master registration for 127.2.215.190:39633: OK, instance_id { permanent_uuid: "eae354d6bcb347a680e647222635cc98" instance_seqno: 1790364796896562 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:33:21.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.298503 2992 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:33:21.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.320369 3040 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [term 1 FOLLOWER]: Refusing update from remote peer 99b8d4b23d8349a2bfd609f2f81e974f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:33:21.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.321911 3154 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [LEADER]: Connected to new peer: Peer: permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:33:21.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.353866 3185 mvcc.cc:205] Tried to move back new op lower bound from 7333334226185793536 to 7333334225800224768. Current Snapshot: MvccSnapshot[applied={T|T < 7333334226185793536}]
19:33:21.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.347949 3169 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 99b8d4b23d8349a2bfd609f2f81e974f. Latest consensus state: current_term: 1 leader_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } } }
19:33:21.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.356530 3169 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [sys.catalog]: This master's current role is: FOLLOWER
19:33:21.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.368496 3154 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } } }
19:33:21.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.369971 3154 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [sys.catalog]: This master's current role is: LEADER
19:33:21.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.371361 3169 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } } }
19:33:21.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.372074 3169 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4 [sys.catalog]: This master's current role is: FOLLOWER
19:33:21.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.376953 3184 catalog_manager.cc:1347] Generated new cluster ID: 8d47abf1ac824581a12874f55e309f77
19:33:21.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.377233 3184 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
19:33:21.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.421411 3184 catalog_manager.cc:1370] Generated new certificate authority record
19:33:21.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.426923 3184 catalog_manager.cc:1504] Loading token signing keys...
19:33:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.460103 3184 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f: Generated new TSK 0
19:33:21.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.462059 3184 catalog_manager.cc:1514] Initializing in-progress tserver states...
19:33:21.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.501300 2910 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/wal
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/data
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/logs
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/data/info.pb
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:33:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:33:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.215.129:0
19:33:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.215.129
19:33:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.215.129
19:33:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.215.190:39633,127.2.215.189:34297,127.2.215.188:46807
19:33:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.215.148:34939
19:33:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:33:21.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.555550 2990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } attempt: 7
19:33:21.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.622253 2990 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98: Bootstrap starting.
19:33:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.680267 2990 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98: Neither blocks nor log segments found. Creating new log.
19:33:21.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.698094 2990 log.cc:824] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98: Log is configured to *not* fsync() on all Append() calls
19:33:21.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.735138 2990 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98: No bootstrap required, opened a new log
19:33:21.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.845022 2990 raft_consensus.cc:348] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } }
19:33:21.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.856482 2990 raft_consensus.cc:374] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:33:21.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.857268 2990 raft_consensus.cc:729] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eae354d6bcb347a680e647222635cc98, State: Initialized, Role: FOLLOWER
19:33:21.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.858582 2990 consensus_queue.cc:260] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } }
19:33:21.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.875144 2990 sys_catalog.cc:564] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [sys.catalog]: configured and running, proceeding with master startup.
19:33:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.876518 2964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eae354d6bcb347a680e647222635cc98"
19:33:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.877215 2964 raft_consensus.cc:3037] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [term 0 FOLLOWER]: Advancing to term 1
19:33:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.871601 3193 sys_catalog.cc:455] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } } }
19:33:21.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.879912 3193 sys_catalog.cc:458] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [sys.catalog]: This master's current role is: FOLLOWER
19:33:21.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.875331 2965 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eae354d6bcb347a680e647222635cc98" is_pre_election: true
19:33:21.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.881770 2965 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:51096 (request call id 2) took 1011 ms. Trace:
19:33:21.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.882071 2965 rpcz_store.cc:276] 0925 19:33:20.870429 (+ 0us) service_pool.cc:170] Inserting onto call queue
19:33:21.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0925 19:33:20.870585 (+ 156us) service_pool.cc:227] Handling call
19:33:21.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0925 19:33:21.881702 (+1011117us) inbound_call.cc:173] Queueing success response
19:33:21.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
19:33:21.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.887677 2964 raft_consensus.cc:2454] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 99b8d4b23d8349a2bfd609f2f81e974f in term 1.
19:33:21.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.889197 2963 raft_consensus.cc:1261] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [term 1 FOLLOWER]: Refusing update from remote peer 99b8d4b23d8349a2bfd609f2f81e974f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:33:21.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:21.897764 3154 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:33:22.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.001470 3193 sys_catalog.cc:455] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 99b8d4b23d8349a2bfd609f2f81e974f. Latest consensus state: current_term: 1 leader_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } } }
19:33:22.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.001641 3203 sys_catalog.cc:455] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } } }
19:33:22.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.003348 3193 sys_catalog.cc:458] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [sys.catalog]: This master's current role is: FOLLOWER
19:33:22.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.003403 3203 sys_catalog.cc:458] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98 [sys.catalog]: This master's current role is: FOLLOWER
19:33:22.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.031941 3211 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
19:33:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:22.094851 3190 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:33:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:22.095404 3190 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:33:22.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:22.096035 3190 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:33:22.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:22.129127 3190 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:33:22.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:22.130143 3190 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.215.129
19:33:22.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.165664 3190 tablet_server_runner.cc:78] Tablet server non-default flags:
19:33:22.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:22.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.215.148:34939
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/data
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/wal
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.215.129:0
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/data/info.pb
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.215.129
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.215.190:39633,127.2.215.189:34297,127.2.215.188:46807
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.215.129
19:33:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/logs
19:33:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:33:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:33:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:33:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:33:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:33:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:33:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 18:42:06 UTC on ccycloud.quasar-gwconk.root.comops.site
19:33:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673102
19:33:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:33:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.167135 3190 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:33:22.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.169117 3190 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:33:22.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:22.195663 3216 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:22.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.255873 3181 catalog_manager.cc:1259] Loaded cluster ID: 8d47abf1ac824581a12874f55e309f77
19:33:22.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.256513 3181 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4: loading cluster ID for follower catalog manager: success
19:33:22.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.265094 3181 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4: acquiring CA information for follower catalog manager: success
19:33:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.284583 3181 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 83f8f3ba27ba41ac9fd6ddb3b762e9b4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:33:22.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.490149 3154 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eae354d6bcb347a680e647222635cc98" member_type: VOTER last_known_addr { host: "127.2.215.190" port: 39633 } } peers { permanent_uuid: "83f8f3ba27ba41ac9fd6ddb3b762e9b4" member_type: VOTER last_known_addr { host: "127.2.215.189" port: 34297 } } peers { permanent_uuid: "99b8d4b23d8349a2bfd609f2f81e974f" member_type: VOTER last_known_addr { host: "127.2.215.188" port: 46807 } } }
19:33:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:22.493605 3154 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99b8d4b23d8349a2bfd609f2f81e974f [sys.catalog]: This master's current role is: LEADER
19:33:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:23.042930 3211 catalog_manager.cc:1259] Loaded cluster ID: 8d47abf1ac824581a12874f55e309f77
19:33:23.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:23.045871 3211 catalog_manager.cc:1552] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98: loading cluster ID for follower catalog manager: success
19:33:23.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:23.073949 3211 catalog_manager.cc:1574] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98: acquiring CA information for follower catalog manager: success
19:33:23.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:23.097994 3211 catalog_manager.cc:1602] T 00000000000000000000000000000000 P eae354d6bcb347a680e647222635cc98: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:33:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:22.197068 3215 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:22.208232 3218 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:24.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.273994 3190 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:33:24.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:24.274497 3217 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2074 milliseconds
19:33:24.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.275887 3190 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:33:24.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.278764 3190 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:33:24.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.280185 3190 hybrid_clock.cc:648] HybridClock initialized: now 1790364804280233 us; error 142 us; skew 500 ppm
19:33:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.281080 3190 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:33:24.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.288975 3190 webserver.cc:460] Webserver started at http://127.2.215.129:43699/ using document root <none> and password file <none>
19:33:24.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.290050 3190 fs_manager.cc:359] Metadata directory not provided
19:33:24.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.290288 3190 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:33:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.291004 3190 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:33:24.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.296891 3190 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/data/instance:
19:33:24.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afaacbf782894d7295fec5afeca213fe"
19:33:24.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:24 on dist-test-slave-555z"
19:33:24.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.298127 3190 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/wal/instance:
19:33:24.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afaacbf782894d7295fec5afeca213fe"
19:33:24.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:24 on dist-test-slave-555z"
19:33:24.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.305744 3190 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
19:33:24.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.312979 3230 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:33:24.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.314145 3190 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
19:33:24.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.314472 3190 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/data,/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/wal
19:33:24.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afaacbf782894d7295fec5afeca213fe"
19:33:24.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:24 on dist-test-slave-555z"
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.314822 3190 fs_report.cc:383] FS layout report
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/wal
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/wal
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/data/data
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:33:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:33:24.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:33:24.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:33:24.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.364741 3190 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:33:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.366664 3190 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:33:24.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.367215 3190 kserver.cc:163] Server-wide thread pool size limit: 3276
19:33:24.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.370573 3190 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:33:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.380846 3190 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:33:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.381115 3190 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:33:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.381412 3190 ts_tablet_manager.cc:610] Registered 0 tablets
19:33:24.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.381637 3190 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:33:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.620651 3190 rpc_server.cc:307] RPC server started. Bound to: 127.2.215.129:34633
19:33:24.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.620743 3342 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.215.129:34633 every 8 connection(s)
19:33:24.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.624063 3190 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/data/info.pb
19:33:24.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.630825 2910 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu as pid 3190
19:33:24.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.631408 2910 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-0/wal/instance
19:33:24.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.665172 2910 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:24.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/wal
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/data
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/logs
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/data/info.pb
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:33:24.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:33:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:33:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.215.130:0
19:33:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.215.130
19:33:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.215.130
19:33:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.215.190:39633,127.2.215.189:34297,127.2.215.188:46807
19:33:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.215.148:34939
19:33:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:33:24.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.732302 3343 heartbeater.cc:347] Connected to a master server at 127.2.215.189:34297
19:33:24.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.732769 3343 heartbeater.cc:464] Registering TS with master...
19:33:24.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.733870 3343 heartbeater.cc:511] Master 127.2.215.189:34297 requested a full tablet report, sending...
19:33:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.737972 3030 ts_manager.cc:194] Registered new tserver with Master: afaacbf782894d7295fec5afeca213fe (127.2.215.129:34633)
19:33:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.743119 3344 heartbeater.cc:347] Connected to a master server at 127.2.215.188:46807
19:33:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.743413 3344 heartbeater.cc:464] Registering TS with master...
19:33:24.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.744195 3344 heartbeater.cc:511] Master 127.2.215.188:46807 requested a full tablet report, sending...
19:33:24.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.747458 3108 ts_manager.cc:194] Registered new tserver with Master: afaacbf782894d7295fec5afeca213fe (127.2.215.129:34633)
19:33:24.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.750806 3108 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.215.129:47341
19:33:24.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.759174 3347 heartbeater.cc:347] Connected to a master server at 127.2.215.190:39633
19:33:24.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.759534 3347 heartbeater.cc:464] Registering TS with master...
19:33:24.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.760207 3347 heartbeater.cc:511] Master 127.2.215.190:39633 requested a full tablet report, sending...
19:33:24.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:24.763612 2955 ts_manager.cc:194] Registered new tserver with Master: afaacbf782894d7295fec5afeca213fe (127.2.215.129:34633)
19:33:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:25.033313 3351 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:33:25.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:25.033865 3351 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:33:25.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:25.034847 3351 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:33:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:25.066823 3351 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:33:25.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:25.067622 3351 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.215.130
19:33:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:25.104192 3351 tablet_server_runner.cc:78] Tablet server non-default flags:
19:33:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.215.148:34939
19:33:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/data
19:33:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/wal
19:33:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.215.130:0
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/data/info.pb
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.215.130
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.215.190:39633,127.2.215.189:34297,127.2.215.188:46807
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:33:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.215.130
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/logs
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 18:42:06 UTC on ccycloud.quasar-gwconk.root.comops.site
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673102
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:25.105362 3351 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:33:25.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:25.107295 3351 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:33:25.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:25.145955 3361 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:25.755426 3344 heartbeater.cc:503] Master 127.2.215.188:46807 was elected leader, sending a full tablet report...
19:33:26.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:25.147116 3363 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:25.146124 3360 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:26.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:26.540669 3362 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1396 milliseconds
19:33:26.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.540760 3351 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:33:26.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.542094 3351 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:33:26.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.544816 3351 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:33:26.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.546231 3351 hybrid_clock.cc:648] HybridClock initialized: now 1790364806546191 us; error 60 us; skew 500 ppm
19:33:26.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.547062 3351 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:33:26.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.559669 3351 webserver.cc:460] Webserver started at http://127.2.215.130:33237/ using document root <none> and password file <none>
19:33:26.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.560628 3351 fs_manager.cc:359] Metadata directory not provided
19:33:26.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.560817 3351 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:33:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.561275 3351 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:33:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.565932 3351 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/data/instance:
19:33:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a069a089f2bb48bca7ed241f0a510518"
19:33:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:26 on dist-test-slave-555z"
19:33:26.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.567154 3351 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/wal/instance:
19:33:26.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a069a089f2bb48bca7ed241f0a510518"
19:33:26.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:26 on dist-test-slave-555z"
19:33:26.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.574622 3351 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
19:33:26.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.580680 3373 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:33:26.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.581879 3351 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
19:33:26.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.582180 3351 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/data,/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/wal
19:33:26.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a069a089f2bb48bca7ed241f0a510518"
19:33:26.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:26 on dist-test-slave-555z"
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.582475 3351 fs_report.cc:383] FS layout report
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/wal
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/wal
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/data/data
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:33:26.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:33:26.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.623405 3351 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:33:26.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.624984 3351 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:33:26.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.625388 3351 kserver.cc:163] Server-wide thread pool size limit: 3276
19:33:26.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.628868 3351 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:33:26.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.633558 3351 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:33:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.633793 3351 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:33:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.634111 3351 ts_tablet_manager.cc:610] Registered 0 tablets
19:33:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.634465 3351 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
19:33:26.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.795571 3351 rpc_server.cc:307] RPC server started. Bound to: 127.2.215.130:35087
19:33:26.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.795706 3485 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.215.130:35087 every 8 connection(s)
19:33:26.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.798326 3351 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/data/info.pb
19:33:26.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.808787 2910 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu as pid 3351
19:33:26.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.809604 2910 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-1/wal/instance
19:33:26.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.833411 2910 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/wal
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/data
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/logs
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/data/info.pb
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:26.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:33:26.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:33:26.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.215.131:0
19:33:26.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.215.131
19:33:26.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.215.131
19:33:26.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:26.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.215.190:39633,127.2.215.189:34297,127.2.215.188:46807
19:33:26.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.215.148:34939
19:33:26.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:26.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:26.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:33:26.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.852566 3486 heartbeater.cc:347] Connected to a master server at 127.2.215.189:34297
19:33:26.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.853067 3486 heartbeater.cc:464] Registering TS with master...
19:33:26.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.854605 3486 heartbeater.cc:511] Master 127.2.215.189:34297 requested a full tablet report, sending...
19:33:26.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.857995 3030 ts_manager.cc:194] Registered new tserver with Master: a069a089f2bb48bca7ed241f0a510518 (127.2.215.130:35087)
19:33:26.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.860842 3487 heartbeater.cc:347] Connected to a master server at 127.2.215.188:46807
19:33:26.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.861248 3487 heartbeater.cc:464] Registering TS with master...
19:33:26.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.862203 3487 heartbeater.cc:511] Master 127.2.215.188:46807 requested a full tablet report, sending...
19:33:26.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.862425 3488 heartbeater.cc:347] Connected to a master server at 127.2.215.190:39633
19:33:26.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.862779 3488 heartbeater.cc:464] Registering TS with master...
19:33:26.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.863481 3488 heartbeater.cc:511] Master 127.2.215.190:39633 requested a full tablet report, sending...
19:33:26.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.864549 3108 ts_manager.cc:194] Registered new tserver with Master: a069a089f2bb48bca7ed241f0a510518 (127.2.215.130:35087)
19:33:26.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.866083 2955 ts_manager.cc:194] Registered new tserver with Master: a069a089f2bb48bca7ed241f0a510518 (127.2.215.130:35087)
19:33:26.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:26.866475 3108 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.215.130:47263
19:33:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:27.166798 3494 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:33:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:27.167364 3494 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:33:27.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:27.167980 3494 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:33:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:27.199520 3494 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:33:27.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:27.200641 3494 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.215.131
19:33:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:27.237222 3494 tablet_server_runner.cc:78] Tablet server non-default flags:
19:33:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.215.148:34939
19:33:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/data
19:33:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/wal
19:33:27.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.215.131:0
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/data/info.pb
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.215.131
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.215.190:39633,127.2.215.189:34297,127.2.215.188:46807
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.215.131
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/logs
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 18:42:06 UTC on ccycloud.quasar-gwconk.root.comops.site
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673102
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:33:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:27.238601 3494 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:33:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:27.240960 3494 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:33:27.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:27.261734 3504 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:27.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:27.870440 3487 heartbeater.cc:503] Master 127.2.215.188:46807 was elected leader, sending a full tablet report...
19:33:28.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:27.261727 3503 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:28.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:27.262933 3506 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:28.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:27.263248 3494 server_base.cc:1034] running on GCE node
19:33:28.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.458508 3494 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:33:28.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.460777 3494 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.462191 3494 hybrid_clock.cc:648] HybridClock initialized: now 1790364808462155 us; error 40 us; skew 500 ppm
19:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.463032 3494 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:33:28.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.469327 3494 webserver.cc:460] Webserver started at http://127.2.215.131:38539/ using document root <none> and password file <none>
19:33:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.470316 3494 fs_manager.cc:359] Metadata directory not provided
19:33:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.470561 3494 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:33:28.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.471092 3494 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:33:28.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.476336 3494 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/data/instance:
19:33:28.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "387a25e809d241efa710e2e711d273f0"
19:33:28.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:28 on dist-test-slave-555z"
19:33:28.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.477593 3494 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/wal/instance:
19:33:28.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "387a25e809d241efa710e2e711d273f0"
19:33:28.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:28 on dist-test-slave-555z"
19:33:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.484949 3494 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
19:33:28.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.490841 3513 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:33:28.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.492024 3494 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
19:33:28.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.492403 3494 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/data,/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/wal
19:33:28.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "387a25e809d241efa710e2e711d273f0"
19:33:28.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:28 on dist-test-slave-555z"
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.492758 3494 fs_report.cc:383] FS layout report
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/wal
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/wal
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/data/data
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:33:28.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:33:28.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.531337 3494 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:33:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.532964 3494 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:33:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.533475 3494 kserver.cc:163] Server-wide thread pool size limit: 3276
19:33:28.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.536463 3494 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:33:28.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.541738 3494 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:33:28.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.541982 3494 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:33:28.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.542301 3494 ts_tablet_manager.cc:610] Registered 0 tablets
19:33:28.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.542482 3494 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:33:28.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.703028 3494 rpc_server.cc:307] RPC server started. Bound to: 127.2.215.131:42231
19:33:28.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.703120 3625 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.215.131:42231 every 8 connection(s)
19:33:28.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.705768 3494 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/data/info.pb
19:33:28.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.709724 2910 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu as pid 3494
19:33:28.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.710173 2910 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster6879197437175779458/ts-2/wal/instance
19:33:28.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.766546 3627 heartbeater.cc:347] Connected to a master server at 127.2.215.188:46807
19:33:28.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.767010 3627 heartbeater.cc:464] Registering TS with master...
19:33:28.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.768122 3627 heartbeater.cc:511] Master 127.2.215.188:46807 requested a full tablet report, sending...
19:33:28.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.768188 3626 heartbeater.cc:347] Connected to a master server at 127.2.215.189:34297
19:33:28.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.768564 3626 heartbeater.cc:464] Registering TS with master...
19:33:28.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.769388 3626 heartbeater.cc:511] Master 127.2.215.189:34297 requested a full tablet report, sending...
19:33:28.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.770660 3108 ts_manager.cc:194] Registered new tserver with Master: 387a25e809d241efa710e2e711d273f0 (127.2.215.131:42231)
19:33:28.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.772667 3108 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.215.131:40667
19:33:28.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.773020 3030 ts_manager.cc:194] Registered new tserver with Master: 387a25e809d241efa710e2e711d273f0 (127.2.215.131:42231)
19:33:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.781828 3628 heartbeater.cc:347] Connected to a master server at 127.2.215.190:39633
19:33:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.782194 3628 heartbeater.cc:464] Registering TS with master...
19:33:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.782857 3628 heartbeater.cc:511] Master 127.2.215.190:39633 requested a full tablet report, sending...
19:33:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.785073 2955 ts_manager.cc:194] Registered new tserver with Master: 387a25e809d241efa710e2e711d273f0 (127.2.215.131:42231)
19:33:28.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:28.798269 2910 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
19:33:28.810 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:33:28.811 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
19:33:28.821 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.215.190"
port: 39633
}
bound_http_address {
host: "127.2.215.190"
port: 40001
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.215.189"
port: 34297
}
bound_http_address {
host: "127.2.215.189"
port: 34933
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.215.188"
port: 46807
}
bound_http_address {
host: "127.2.215.188"
port: 36235
}
}
}
19:33:28.826 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
19:33:28.828 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.2.215.129"
port: 34633
}
bound_http_address {
host: "127.2.215.129"
port: 43699
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.2.215.130"
port: 35087
}
bound_http_address {
host: "127.2.215.130"
port: 33237
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.2.215.131"
port: 42231
}
bound_http_address {
host: "127.2.215.131"
port: 38539
}
}
}
19:33:28.828 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
19:33:29.013 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
19:33:29.146 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQtGm0f/test-tmp
19:33:29.147 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
19:33:29.148 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQtGm0f/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
19:33:29.150 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
19:33:29.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:33:29.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
19:33:29.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:29.512084 3638 mini_oidc.cc:152] Starting JWKS server
19:33:29.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:29.516090 3638 webserver.cc:460] Webserver started at http://127.0.0.1:46737/ using document root <none> and password file <none>
19:33:29.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:29.516741 3638 mini_oidc.cc:189] Starting OIDC Discovery server
19:33:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:29.518350 3638 webserver.cc:460] Webserver started at http://127.0.0.1:39613/ using document root <none> and password file <none>
19:33:29.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
19:33:29.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/principal' for realm 'KRBTEST.COM',
19:33:29.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
19:33:29.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:29 dist-test-slave-555z krb5kdc[3650](info): setting up network...
19:33:29.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
19:33:29.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:29 dist-test-slave-555z krb5kdc[3650](info): set up 2 sockets
19:33:29.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:29 dist-test-slave-555z krb5kdc[3650](info): commencing operation
19:33:29.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
19:33:29.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:29.780970 3627 heartbeater.cc:503] Master 127.2.215.188:46807 was elected leader, sending a full tablet report...
19:33:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:30.658618 3638 mini_kdc.cc:121] Time spent starting KDC: real 1.116s user 0.004s sys 0.016s
19:33:30.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
19:33:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:33:30.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
19:33:30.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
19:33:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:33:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
19:33:30.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
19:33:30.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:33:30.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
19:33:30.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:33:30.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/test-user.keytab.
19:33:30.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/test-user.keytab.
19:33:30.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:30 dist-test-slave-555z krb5kdc[3650](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364810, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:33:30.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:33:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T19:33:30Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
19:33:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T19:33:30Z Disabled control of system clock
19:33:30.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.3.141.190@KRBTEST.COM; defaulting to no policy
19:33:30.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:33:30.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.3.141.190@KRBTEST.COM" created.
19:33:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:33:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.141.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.190.keytab.
19:33:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.141.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.190.keytab.
19:33:30.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:30.868211 3638 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:30.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:30.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal
19:33:30.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data
19:33:30.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:33:30.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:33:30.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/logs
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data/info.pb
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.141.190:35831
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.141.190
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.190.keytab
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.141.190
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:33:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.141.148:36523
19:33:30.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:30.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:30.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:30.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:30.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.141.190:35831,127.3.141.189:40587,127.3.141.188:33891
19:33:30.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:30.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46737/jwks.json
19:33:30.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:33:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:31.194188 3663 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:33:31.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:31.194818 3663 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:33:31.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:31.195402 3663 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:33:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:31.226967 3663 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:33:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:31.227210 3663 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:33:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:31.227583 3663 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:33:31.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:31.227793 3663 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:33:31.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:31.228040 3663 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:33:31.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:31.228237 3663 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:33:31.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:31.228514 3663 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46737/jwks.json
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:31.264925 3663 master_runner.cc:386] Master server non-default flags:
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.141.148:36523
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.141.190:35831,127.3.141.189:40587,127.3.141.188:33891
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.190.keytab
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.141.190
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.141.190:35831
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46737/jwks.json
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data/info.pb
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.141.190
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/logs
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 18:42:06 UTC on ccycloud.quasar-gwconk.root.comops.site
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673102
19:33:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:33:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:31.266398 3663 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:33:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:31.267560 3663 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:33:31.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:31.286108 3669 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:31.286309 3670 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:31.286612 3672 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:31.288758 3663 server_base.cc:1034] running on GCE node
19:33:32.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.510004 3663 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:33:32.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.513109 3663 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:33:32.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.514525 3663 hybrid_clock.cc:648] HybridClock initialized: now 1790364812514502 us; error 58 us; skew 500 ppm
19:33:32.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.515312 3663 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:33:32.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:32 dist-test-slave-555z krb5kdc[3650](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364812, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:33:32.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.525481 3663 init.cc:377] Logged in from keytab as kudu/127.3.141.190@KRBTEST.COM (short username kudu)
19:33:32.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.531402 3663 webserver.cc:460] Webserver started at http://127.3.141.190:37577/ using document root <none> and password file <none>
19:33:32.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.532325 3663 fs_manager.cc:359] Metadata directory not provided
19:33:32.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.532577 3663 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:33:32.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.533090 3663 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:33:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.537650 3663 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data/instance:
19:33:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbbc07deb8414afc8aa0535445f6b7a9"
19:33:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:32 on dist-test-slave-555z"
19:33:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.538784 3663 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal/instance:
19:33:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbbc07deb8414afc8aa0535445f6b7a9"
19:33:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:32 on dist-test-slave-555z"
19:33:32.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.546718 3663 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
19:33:32.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.552341 3681 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:33:32.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.553591 3663 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
19:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.553890 3663 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data,/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal
19:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbbc07deb8414afc8aa0535445f6b7a9"
19:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:32 on dist-test-slave-555z"
19:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.554190 3663 fs_report.cc:383] FS layout report
19:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal
19:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal
19:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data/data
19:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:33:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:33:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:33:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:33:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:33:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:33:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:33:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:33:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.590271 3663 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:33:32.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.596567 3663 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:33:32.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.596967 3663 kserver.cc:163] Server-wide thread pool size limit: 3276
19:33:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.678843 3663 rpc_server.cc:307] RPC server started. Bound to: 127.3.141.190:35831
19:33:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.678918 3732 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.141.190:35831 every 8 connection(s)
19:33:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.681749 3663 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data/info.pb
19:33:32.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.687471 3733 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:33:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.691045 3638 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu as pid 3663
19:33:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.691449 3638 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal/instance
19:33:32.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.700438 3733 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } has no permanent_uuid. Determining permanent_uuid...
19:33:32.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.3.141.189@KRBTEST.COM; defaulting to no policy
19:33:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:33:32.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.3.141.189@KRBTEST.COM" created.
19:33:32.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:32 dist-test-slave-555z krb5kdc[3650](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364812, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.190@KRBTEST.COM for kudu/127.3.141.190@KRBTEST.COM
19:33:32.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:33:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.141.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.189.keytab.
19:33:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.141.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.189.keytab.
19:33:32.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.745249 3638 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:32.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:32.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal
19:33:32.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data
19:33:32.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:33:32.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:33:32.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:33:32.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:32.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:32.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/logs
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data/info.pb
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.141.189:40587
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.141.189
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.189.keytab
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.141.189
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.141.148:36523
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.141.190:35831,127.3.141.189:40587,127.3.141.188:33891
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46737/jwks.json
19:33:32.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:33:32.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.753234 3733 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } has no permanent_uuid. Determining permanent_uuid...
19:33:32.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:32.755599 3685 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:33:32.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:32.759666 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.189:40587: Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:33:32.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.809243 3733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } attempt: 1
19:33:32.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:32.813071 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.189:40587: Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:33:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.881546 3733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } attempt: 2
19:33:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:32.885152 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.189:40587: Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:33:32.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:32.976557 3733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } attempt: 3
19:33:32.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:32.980520 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.189:40587: Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:33:33.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.071698 3739 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:33:33.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.072331 3739 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:33:33.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.073076 3739 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:33:33.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.107869 3739 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:33:33.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.108111 3739 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:33:33.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.108479 3739 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:33:33.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.108691 3739 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:33:33.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.108876 3739 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:33:33.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.109068 3739 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:33:33.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.109206 3739 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46737/jwks.json
19:33:33.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:33.123911 3733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } attempt: 4
19:33:33.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.127511 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.189:40587: Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:33:33.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:33.147511 3739 master_runner.cc:386] Master server non-default flags:
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.141.148:36523
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.141.190:35831,127.3.141.189:40587,127.3.141.188:33891
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.189.keytab
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.141.189
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.141.189:40587
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46737/jwks.json
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data/info.pb
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.141.189
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/logs
19:33:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:33:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:33:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:33:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:33:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:33:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:33:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 18:42:06 UTC on ccycloud.quasar-gwconk.root.comops.site
19:33:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673102
19:33:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:33:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:33.148715 3739 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:33:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:33.149969 3739 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:33:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.169827 3747 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:33.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:33.426932 3733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } attempt: 5
19:33:33.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.431941 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.189:40587: Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:33:33.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:33.979308 3733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } attempt: 6
19:33:33.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.985631 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.189:40587: Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:33:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.169829 3749 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:33.169852 3746 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:34.406227 3748 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
19:33:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.406563 3739 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:33:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.410181 3739 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:33:34.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.413003 3739 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:33:34.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.414420 3739 hybrid_clock.cc:648] HybridClock initialized: now 1790364814414397 us; error 68 us; skew 500 ppm
19:33:34.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.415248 3739 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:33:34.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:34 dist-test-slave-555z krb5kdc[3650](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364814, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:33:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.420868 3739 init.cc:377] Logged in from keytab as kudu/127.3.141.189@KRBTEST.COM (short username kudu)
19:33:34.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.426208 3739 webserver.cc:460] Webserver started at http://127.3.141.189:43355/ using document root <none> and password file <none>
19:33:34.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.427103 3739 fs_manager.cc:359] Metadata directory not provided
19:33:34.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.427284 3739 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:33:34.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.427651 3739 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:33:34.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.433343 3739 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data/instance:
19:33:34.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06f5921fd8e344ffac65d431b5adad19"
19:33:34.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:34 on dist-test-slave-555z"
19:33:34.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.434459 3739 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal/instance:
19:33:34.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06f5921fd8e344ffac65d431b5adad19"
19:33:34.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:34 on dist-test-slave-555z"
19:33:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.441538 3739 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
19:33:34.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.447460 3759 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:33:34.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.448657 3739 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
19:33:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.448951 3739 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data,/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal
19:33:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06f5921fd8e344ffac65d431b5adad19"
19:33:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:34 on dist-test-slave-555z"
19:33:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.449256 3739 fs_report.cc:383] FS layout report
19:33:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:33:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal
19:33:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal
19:33:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data/data
19:33:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:33:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:33:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:33:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:33:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:33:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:33:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:33:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:33:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:33:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:33:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:33:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.495373 3739 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:33:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.501582 3739 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:33:34.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.501977 3739 kserver.cc:163] Server-wide thread pool size limit: 3276
19:33:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.582895 3739 rpc_server.cc:307] RPC server started. Bound to: 127.3.141.189:40587
19:33:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.582942 3810 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.141.189:40587 every 8 connection(s)
19:33:34.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.586669 3739 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data/info.pb
19:33:34.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.593214 3638 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu as pid 3739
19:33:34.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.593353 3811 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:33:34.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.593690 3638 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal/instance
19:33:34.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.608385 3811 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } has no permanent_uuid. Determining permanent_uuid...
19:33:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.3.141.188@KRBTEST.COM; defaulting to no policy
19:33:34.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:33:34.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.3.141.188@KRBTEST.COM" created.
19:33:34.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:34 dist-test-slave-555z krb5kdc[3650](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364814, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.189@KRBTEST.COM for kudu/127.3.141.190@KRBTEST.COM
19:33:34.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:33:34.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.141.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.188.keytab.
19:33:34.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.141.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.188.keytab.
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.642827 3638 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/wal
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/logs
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data/info.pb
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:34.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:33:34.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:33:34.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.141.188:33891
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.141.188
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.188.keytab
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.141.188
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.141.148:36523
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.141.190:35831,127.3.141.189:40587,127.3.141.188:33891
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46737/jwks.json
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:33:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.645532 3811 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } has no permanent_uuid. Determining permanent_uuid...
19:33:34.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:34 dist-test-slave-555z krb5kdc[3650](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364814, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.189@KRBTEST.COM for kudu/127.3.141.189@KRBTEST.COM
19:33:34.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.675204 3811 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } has no permanent_uuid. Determining permanent_uuid...
19:33:34.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:34.678016 3761 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:34.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:34.682472 3811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.731963 3811 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } attempt: 1
19:33:34.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:34.736768 3811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:34.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.805169 3811 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } attempt: 2
19:33:34.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:34.809535 3811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:34.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:34.900902 3811 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } attempt: 3
19:33:34.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:34.904543 3811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:34.982861 3818 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:33:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:34.983523 3818 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:33:34.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:34.984225 3818 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:33:35.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.016259 3818 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:33:35.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.016599 3818 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:33:35.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.017110 3818 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:33:35.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.017457 3818 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:33:35.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.017776 3818 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:33:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.018114 3818 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:33:35.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.018496 3818 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46737/jwks.json
19:33:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.045979 3733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } attempt: 7
19:33:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.048039 3811 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } attempt: 4
19:33:35.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.051577 3811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:35.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:35 dist-test-slave-555z krb5kdc[3650](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364812, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.190@KRBTEST.COM for kudu/127.3.141.189@KRBTEST.COM
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.063735 3818 master_runner.cc:386] Master server non-default flags:
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.141.148:36523
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/wal
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.141.190:35831,127.3.141.189:40587,127.3.141.188:33891
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.188.keytab
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.141.188
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.141.188:33891
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46737/jwks.json
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:33:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data/info.pb
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.141.188
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/logs
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 18:42:06 UTC on ccycloud.quasar-gwconk.root.comops.site
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673102
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:33:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.064945 3818 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:33:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.066129 3818 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:33:35.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.071763 3733 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } has no permanent_uuid. Determining permanent_uuid...
19:33:35.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.078841 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.088383 3829 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:35.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.137356 3733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } attempt: 1
19:33:35.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.141396 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:35.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.223001 3733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } attempt: 2
19:33:35.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.227145 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:35.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.312774 3733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } attempt: 3
19:33:35.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.316879 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:35.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.351017 3811 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } attempt: 5
19:33:35.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.355876 3811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:35.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.457525 3733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } attempt: 4
19:33:35.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.461503 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.745115 3733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } attempt: 5
19:33:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.749207 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:35.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.903565 3811 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } attempt: 6
19:33:35.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.910351 3811 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.301818 3733 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } attempt: 6
19:33:36.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:36.307854 3733 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.141.188:33891: Network error: Client connection negotiation failed: client connection to 127.3.141.188:33891: connect: Connection refused (error 111)
19:33:36.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:35.089875 3818 server_base.cc:1034] running on GCE node
19:33:36.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.091572 3832 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:36.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:35.091861 3830 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:36.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.366811 3818 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:33:36.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.369776 3818 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:33:36.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.371115 3818 hybrid_clock.cc:648] HybridClock initialized: now 1790364816371093 us; error 46 us; skew 500 ppm
19:33:36.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.371896 3818 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:33:36.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:36 dist-test-slave-555z krb5kdc[3650](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364816, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:33:36.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.377803 3818 init.cc:377] Logged in from keytab as kudu/127.3.141.188@KRBTEST.COM (short username kudu)
19:33:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.383190 3818 webserver.cc:460] Webserver started at http://127.3.141.188:43721/ using document root <none> and password file <none>
19:33:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.384097 3818 fs_manager.cc:359] Metadata directory not provided
19:33:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.384274 3818 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:33:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.384732 3818 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:33:36.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.389985 3818 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data/instance:
19:33:36.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e68a8fd751ef475db7d3cb36d7a1667a"
19:33:36.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:36 on dist-test-slave-555z"
19:33:36.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.391199 3818 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/wal/instance:
19:33:36.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e68a8fd751ef475db7d3cb36d7a1667a"
19:33:36.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:36 on dist-test-slave-555z"
19:33:36.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.398401 3818 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
19:33:36.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.404866 3845 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:33:36.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.405876 3818 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
19:33:36.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.406240 3818 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data,/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/wal
19:33:36.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e68a8fd751ef475db7d3cb36d7a1667a"
19:33:36.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:36 on dist-test-slave-555z"
19:33:36.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.406589 3818 fs_report.cc:383] FS layout report
19:33:36.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:33:36.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/wal
19:33:36.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/wal
19:33:36.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data/data
19:33:36.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:33:36.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:33:36.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:33:36.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:33:36.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:33:36.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:33:36.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:33:36.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:33:36.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:33:36.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:33:36.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:33:36.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.442916 3818 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:33:36.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.449481 3818 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:33:36.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.450080 3818 kserver.cc:163] Server-wide thread pool size limit: 3276
19:33:36.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.529139 3818 rpc_server.cc:307] RPC server started. Bound to: 127.3.141.188:33891
19:33:36.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.529223 3896 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.141.188:33891 every 8 connection(s)
19:33:36.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.531985 3818 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data/info.pb
19:33:36.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.539091 3897 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:33:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.540226 3638 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu as pid 3818
19:33:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.540655 3638 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/wal/instance
19:33:36.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.555025 3897 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } has no permanent_uuid. Determining permanent_uuid...
19:33:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:36 dist-test-slave-555z krb5kdc[3650](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364810, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.141.188@KRBTEST.COM
19:33:36.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:36 dist-test-slave-555z krb5kdc[3650](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364816, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.188@KRBTEST.COM for kudu/127.3.141.190@KRBTEST.COM
19:33:36.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.608608 3897 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } has no permanent_uuid. Determining permanent_uuid...
19:33:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:36 dist-test-slave-555z krb5kdc[3650](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364810, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.141.189@KRBTEST.COM
19:33:36.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:36 dist-test-slave-555z krb5kdc[3650](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364816, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.188@KRBTEST.COM for kudu/127.3.141.189@KRBTEST.COM
19:33:36.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.638939 3897 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } has no permanent_uuid. Determining permanent_uuid...
19:33:36.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:36 dist-test-slave-555z krb5kdc[3650](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364810, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.141.190@KRBTEST.COM
19:33:36.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:36 dist-test-slave-555z krb5kdc[3650](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364816, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.188@KRBTEST.COM for kudu/127.3.141.188@KRBTEST.COM
19:33:36.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.668062 3638 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
19:33:36.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.669878 3897 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: Bootstrap starting.
19:33:36.675 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:33:36.676 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
19:33:36.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.677665 3897 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: Neither blocks nor log segments found. Creating new log.
19:33:36.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.679414 3897 log.cc:824] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: Log is configured to *not* fsync() on all Append() calls
19:33:36.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.684736 3897 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: No bootstrap required, opened a new log
19:33:36.694 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
19:33:36.695 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
19:33:36.699 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.141.190"
port: 35831
}
bound_http_address {
host: "127.3.141.190"
port: 37577
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.141.189"
port: 40587
}
bound_http_address {
host: "127.3.141.189"
port: 43355
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.141.188"
port: 33891
}
bound_http_address {
host: "127.3.141.188"
port: 43721
}
}
}
19:33:36.701 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
19:33:36.701 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:33:36.701 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
19:33:36.703 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
19:33:36.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.718113 3897 raft_consensus.cc:348] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbbc07deb8414afc8aa0535445f6b7a9" member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } } peers { permanent_uuid: "06f5921fd8e344ffac65d431b5adad19" member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } } peers { permanent_uuid: "e68a8fd751ef475db7d3cb36d7a1667a" member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } }
19:33:36.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.719030 3897 raft_consensus.cc:374] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:33:36.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.719341 3897 raft_consensus.cc:729] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e68a8fd751ef475db7d3cb36d7a1667a, State: Initialized, Role: FOLLOWER
19:33:36.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.720899 3897 consensus_queue.cc:260] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbbc07deb8414afc8aa0535445f6b7a9" member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } } peers { permanent_uuid: "06f5921fd8e344ffac65d431b5adad19" member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } } peers { permanent_uuid: "e68a8fd751ef475db7d3cb36d7a1667a" member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } }
19:33:36.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.724123 3907 sys_catalog.cc:455] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbbc07deb8414afc8aa0535445f6b7a9" member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } } peers { permanent_uuid: "06f5921fd8e344ffac65d431b5adad19" member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } } peers { permanent_uuid: "e68a8fd751ef475db7d3cb36d7a1667a" member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } } }
19:33:36.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.725126 3907 sys_catalog.cc:458] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [sys.catalog]: This master's current role is: FOLLOWER
19:33:36.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.726166 3897 sys_catalog.cc:564] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [sys.catalog]: configured and running, proceeding with master startup.
19:33:36.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:36 dist-test-slave-555z krb5kdc[3650](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364816, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:33:36.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:33:36.753 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:33:36.755 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5cc
19:33:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:36.764379 3919 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:36.764709 3919 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:36.766 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:33:36.768 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5cc
19:33:36.770 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:33:36.775 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
19:33:36.778 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
19:33:36.779 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
19:33:36.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.754041 3818 master_runner.cc:186] Error getting master registration for 127.3.141.190:35831: OK, instance_id { permanent_uuid: "cbbc07deb8414afc8aa0535445f6b7a9" instance_seqno: 1790364812657254 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:33:36.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.787770 3818 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:33:36.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:36 dist-test-slave-555z krb5kdc[3650](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364816, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:33:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:33:36.817 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:33:36.817 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5cc
19:33:36.819 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:33:36.820 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.141.189:40587
19:33:36.822 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
19:33:36.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.823422 3638 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu with pid 3739
19:33:36.846 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:33:36.846 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.141.190:35831
19:33:36.848 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
19:33:36.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.848872 3638 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu with pid 3663
19:33:36.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:33:36.871 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.141.188:33891
19:33:36.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
19:33:36.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.872145 3638 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu with pid 3818
19:33:36.893 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:33:36.894 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.3.141.189:40587
19:33:36.896 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:36.897761 3638 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/logs
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data/info.pb
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:33:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.141.189:40587
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.141.189
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43355
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.189.keytab
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.141.189
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.141.148:36523
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.141.190:35831,127.3.141.189:40587,127.3.141.188:33891
19:33:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:36.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46737/jwks.json
19:33:36.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:33:37.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:37.223533 3923 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:33:37.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:37.224223 3923 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:33:37.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:37.225041 3923 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:33:37.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:37.258217 3923 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:33:37.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:37.258438 3923 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:33:37.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:37.258690 3923 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:33:37.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:37.258942 3923 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:33:37.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:37.259199 3923 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:33:37.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:37.259469 3923 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:33:37.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:37.259617 3923 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46737/jwks.json
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:37.297767 3923 master_runner.cc:386] Master server non-default flags:
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.141.148:36523
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.141.190:35831,127.3.141.189:40587,127.3.141.188:33891
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.189.keytab
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.141.189
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.141.189:40587
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:33:37.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46737/jwks.json
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data/info.pb
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.141.189
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43355
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/logs
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 18:42:06 UTC on ccycloud.quasar-gwconk.root.comops.site
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673102
19:33:37.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:37.299252 3923 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:33:37.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:37.300673 3923 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:33:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:37.317624 3934 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:38.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:37.319131 3923 server_base.cc:1034] running on GCE node
19:33:38.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:37.318405 3932 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:38.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:37.318049 3931 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:38.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.573374 3923 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:33:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.576551 3923 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:33:38.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.578030 3923 hybrid_clock.cc:648] HybridClock initialized: now 1790364818577950 us; error 52 us; skew 500 ppm
19:33:38.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.578804 3923 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:33:38.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:38 dist-test-slave-555z krb5kdc[3650](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364818, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:33:38.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.592849 3923 init.cc:377] Logged in from keytab as kudu/127.3.141.189@KRBTEST.COM (short username kudu)
19:33:38.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.598083 3923 webserver.cc:460] Webserver started at http://127.3.141.189:43355/ using document root <none> and password file <none>
19:33:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.599116 3923 fs_manager.cc:359] Metadata directory not provided
19:33:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.599326 3923 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:33:38.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.607354 3923 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
19:33:38.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.611507 3942 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:33:38.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.612624 3923 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
19:33:38.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.612957 3923 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data,/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal
19:33:38.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06f5921fd8e344ffac65d431b5adad19"
19:33:38.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:34 on dist-test-slave-555z"
19:33:38.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.615413 3923 fs_report.cc:383] FS layout report
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data/data
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:33:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:33:38.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.652624 3923 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:33:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.657954 3923 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:33:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.658468 3923 kserver.cc:163] Server-wide thread pool size limit: 3276
19:33:38.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.734661 3923 rpc_server.cc:307] RPC server started. Bound to: 127.3.141.189:40587
19:33:38.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.734819 3993 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.141.189:40587 every 8 connection(s)
19:33:38.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.738008 3923 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/data/info.pb
19:33:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.745760 3638 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu as pid 3923
19:33:38.746 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:33:38.746 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.3.141.190:35831
19:33:38.746 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.748575 3638 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/logs
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data/info.pb
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:33:38.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:38.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:38.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.141.190:35831
19:33:38.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.141.190
19:33:38.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37577
19:33:38.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.190.keytab
19:33:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.141.190
19:33:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:33:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:33:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:33:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.141.148:36523
19:33:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.141.190:35831,127.3.141.189:40587,127.3.141.188:33891
19:33:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46737/jwks.json
19:33:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:33:38.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.752990 3994 sys_catalog.cc:263] Verifying existing consensus state
19:33:38.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260925 19:33:38.753989 3994 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:33:38.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.754488 3923 master.cc:537] Master@127.3.141.189:40587 shutting down...
19:33:38.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:38.813874 3923 master.cc:552] Master@127.3.141.189:40587 shutdown complete.
19:33:38.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:33:39.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.074170 3622 debug-util.cc:398] Leaking SignalData structure 0x7b08000bec00 after lost signal to thread 3497
19:33:39.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.075232 3622 debug-util.cc:398] Leaking SignalData structure 0x7b08000bd8e0 after lost signal to thread 3625
19:33:39.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.106696 3996 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:33:39.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.107537 3996 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:33:39.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.108759 3996 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:33:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.157968 3996 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:33:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.158213 3996 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:33:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.158471 3996 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:33:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.158715 3996 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:33:39.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.159036 3996 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:33:39.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.159370 3996 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:33:39.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.159657 3996 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46737/jwks.json
19:33:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:39.199270 3996 master_runner.cc:386] Master server non-default flags:
19:33:39.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.141.148:36523
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.141.190:35831,127.3.141.189:40587,127.3.141.188:33891
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.190.keytab
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.141.190
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.141.190:35831
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46737/jwks.json
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:33:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data/info.pb
19:33:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.141.190
19:33:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37577
19:33:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:33:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:33:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:33:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/logs
19:33:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:33:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:33:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:33:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:33:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:33:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:33:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 18:42:06 UTC on ccycloud.quasar-gwconk.root.comops.site
19:33:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673102
19:33:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:33:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:39.200886 3996 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:33:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:39.202325 3996 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:33:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.223935 4004 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.226332 4006 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:39.224066 4003 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:40.478978 4005 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
19:33:40.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.479265 3996 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:33:40.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.483176 3996 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:33:40.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.486343 3996 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:33:40.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.487776 3996 hybrid_clock.cc:648] HybridClock initialized: now 1790364820487753 us; error 50 us; skew 500 ppm
19:33:40.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.488607 3996 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:33:40.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:40 dist-test-slave-555z krb5kdc[3650](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364820, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:33:40.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.495236 3996 init.cc:377] Logged in from keytab as kudu/127.3.141.190@KRBTEST.COM (short username kudu)
19:33:40.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.501287 3996 webserver.cc:460] Webserver started at http://127.3.141.190:37577/ using document root <none> and password file <none>
19:33:40.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.502343 3996 fs_manager.cc:359] Metadata directory not provided
19:33:40.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.502588 3996 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:33:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.511123 3996 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
19:33:40.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.516129 4015 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:33:40.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.517354 3996 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
19:33:40.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.517624 3996 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data,/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal
19:33:40.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbbc07deb8414afc8aa0535445f6b7a9"
19:33:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:32 on dist-test-slave-555z"
19:33:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.519853 3996 fs_report.cc:383] FS layout report
19:33:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:33:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal
19:33:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal
19:33:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data/data
19:33:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:33:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:33:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:33:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:33:40.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:33:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:33:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:33:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:33:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:33:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:33:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:33:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.564033 3996 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:33:40.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.570226 3996 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:33:40.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.570751 3996 kserver.cc:163] Server-wide thread pool size limit: 3276
19:33:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.668016 3996 rpc_server.cc:307] RPC server started. Bound to: 127.3.141.190:35831
19:33:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.668190 4066 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.141.190:35831 every 8 connection(s)
19:33:40.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.670949 3996 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/data/info.pb
19:33:40.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.678349 3638 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu as pid 3996
19:33:40.679 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:33:40.680 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.3.141.188:33891
19:33:40.680 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
19:33:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.682191 3638 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu
19:33:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/wal
19:33:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data
19:33:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:33:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:33:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:33:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:40.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/logs
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data/info.pb
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.141.188:33891
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.141.188
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43721
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.188.keytab
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.141.188
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.141.148:36523
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.141.190:35831,127.3.141.189:40587,127.3.141.188:33891
19:33:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46737/jwks.json
19:33:40.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:33:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.685019 4067 sys_catalog.cc:263] Verifying existing consensus state
19:33:40.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260925 19:33:40.685997 4067 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:33:40.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.686530 3996 master.cc:537] Master@127.3.141.190:35831 shutting down...
19:33:40.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:40.775794 3996 master.cc:552] Master@127.3.141.190:35831 shutdown complete.
19:33:40.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:33:41.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:41.033939 4069 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:33:41.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:41.034559 4069 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:33:41.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:41.035161 4069 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:33:41.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:41.066037 4069 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:33:41.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:41.066260 4069 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:33:41.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:41.066563 4069 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:33:41.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:41.066825 4069 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:33:41.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:41.067055 4069 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:33:41.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:41.067245 4069 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:33:41.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:41.067391 4069 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46737/jwks.json
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:41.103847 4069 master_runner.cc:386] Master server non-default flags:
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.141.148:36523
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/wal
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.141.190:35831,127.3.141.189:40587,127.3.141.188:33891
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/krb5kdc/kudu_127.3.141.188.keytab
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.141.188
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.141.188:33891
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46737/jwks.json
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data/info.pb
19:33:41.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.141.188
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43721
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/logs
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 18:42:06 UTC on ccycloud.quasar-gwconk.root.comops.site
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673102
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:33:41.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:41.105108 4069 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:33:41.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:41.106310 4069 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:33:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:41.123659 4078 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:42.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:41.124195 4075 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:42.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:41.125777 4069 server_base.cc:1034] running on GCE node
19:33:42.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:41.124734 4076 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:33:42.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.346931 4069 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:33:42.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.349507 4069 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:33:42.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.350835 4069 hybrid_clock.cc:648] HybridClock initialized: now 1790364822350814 us; error 45 us; skew 500 ppm
19:33:42.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.351630 4069 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:33:42.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:42 dist-test-slave-555z krb5kdc[3650](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364822, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:33:42.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.357416 4069 init.cc:377] Logged in from keytab as kudu/127.3.141.188@KRBTEST.COM (short username kudu)
19:33:42.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.362505 4069 webserver.cc:460] Webserver started at http://127.3.141.188:43721/ using document root <none> and password file <none>
19:33:42.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.363384 4069 fs_manager.cc:359] Metadata directory not provided
19:33:42.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.363587 4069 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:33:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.371141 4069 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.003s
19:33:42.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.375396 4087 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:33:42.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.376472 4069 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.376820 4069 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data,/tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/wal
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e68a8fd751ef475db7d3cb36d7a1667a"
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 19:33:36 on dist-test-slave-555z"
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.379077 4069 fs_report.cc:383] FS layout report
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/wal
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/wal
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data/data
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:33:42.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:33:42.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.429569 4069 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:33:42.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.435500 4069 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:33:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.435922 4069 kserver.cc:163] Server-wide thread pool size limit: 3276
19:33:42.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.503702 4069 rpc_server.cc:307] RPC server started. Bound to: 127.3.141.188:33891
19:33:42.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.503819 4138 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.141.188:33891 every 8 connection(s)
19:33:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.506403 4069 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskQtGm0f/test-tmp/mini-kudu-cluster485725336518180783/master-2/data/info.pb
19:33:42.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.508533 3638 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu as pid 4069
19:33:42.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:33:42.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.520368 4139 sys_catalog.cc:263] Verifying existing consensus state
19:33:42.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.528177 4139 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: Bootstrap starting.
19:33:42.567 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:42.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.572113 4139 log.cc:824] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: Log is configured to *not* fsync() on all Append() calls
19:33:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.578603 4139 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:33:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.579463 4139 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: Bootstrap complete.
19:33:42.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.616303 4139 raft_consensus.cc:348] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbbc07deb8414afc8aa0535445f6b7a9" member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } } peers { permanent_uuid: "06f5921fd8e344ffac65d431b5adad19" member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } } peers { permanent_uuid: "e68a8fd751ef475db7d3cb36d7a1667a" member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } }
19:33:42.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.619680 4139 raft_consensus.cc:374] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:33:42.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.619989 4139 raft_consensus.cc:729] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e68a8fd751ef475db7d3cb36d7a1667a, State: Initialized, Role: FOLLOWER
19:33:42.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.621008 4139 consensus_queue.cc:260] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbbc07deb8414afc8aa0535445f6b7a9" member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } } peers { permanent_uuid: "06f5921fd8e344ffac65d431b5adad19" member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } } peers { permanent_uuid: "e68a8fd751ef475db7d3cb36d7a1667a" member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } }
19:33:42.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.639137 4143 sys_catalog.cc:455] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbbc07deb8414afc8aa0535445f6b7a9" member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } } peers { permanent_uuid: "06f5921fd8e344ffac65d431b5adad19" member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } } peers { permanent_uuid: "e68a8fd751ef475db7d3cb36d7a1667a" member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } } }
19:33:42.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.640107 4143 sys_catalog.cc:458] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [sys.catalog]: This master's current role is: FOLLOWER
19:33:42.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.640875 4139 sys_catalog.cc:564] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [sys.catalog]: configured and running, proceeding with master startup.
19:33:42.644 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:42.649 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.141.188:33891(127.3.141.188:33891)] connecting to peer
19:33:42.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:42.680146 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:42.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:42.680744 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:42.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:42.694244 4093 tablet.cc:2369] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:33:42.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.706032 4069 master_runner.cc:186] Error getting master registration for 127.3.141.190:35831: Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111),
19:33:42.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:42.735484 4069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:33:42.747 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:42.747 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:42.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:42.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:42.753 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:42.755 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:42.756 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.141.188:33891(127.3.141.188:33891)] Successfully connected to peer
19:33:42.843 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
19:33:42.852 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.3.141.188
19:33:43.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:43 dist-test-slave-555z krb5kdc[3650](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364816, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.141.188@KRBTEST.COM
19:33:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:43.080518 4143 raft_consensus.cc:482] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:33:43.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:43.081050 4143 raft_consensus.cc:504] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbbc07deb8414afc8aa0535445f6b7a9" member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } } peers { permanent_uuid: "06f5921fd8e344ffac65d431b5adad19" member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } } peers { permanent_uuid: "e68a8fd751ef475db7d3cb36d7a1667a" member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } }
19:33:43.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:43.084116 4143 leader_election.cc:288] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cbbc07deb8414afc8aa0535445f6b7a9 (127.3.141.190:35831), 06f5921fd8e344ffac65d431b5adad19 (127.3.141.189:40587)
19:33:43.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:43.085850 4088 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111)
19:33:43.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:43.096843 4088 leader_election.cc:334] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer cbbc07deb8414afc8aa0535445f6b7a9 (127.3.141.190:35831): Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111)
19:33:43.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:43.102496 4091 leader_election.cc:334] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 06f5921fd8e344ffac65d431b5adad19 (127.3.141.189:40587): Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:33:43.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:43.102895 4091 leader_election.cc:302] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e68a8fd751ef475db7d3cb36d7a1667a; no voters: 06f5921fd8e344ffac65d431b5adad19, cbbc07deb8414afc8aa0535445f6b7a9
19:33:43.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:43.103698 4143 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:33:43.124 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x1880bccf, L:/127.0.0.1:44592 - R:/127.3.141.188:33891] using SASL/GSSAPI
19:33:43.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:43.156 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:43.159 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:43.160 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.161 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.162 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.170 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.170 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.171 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:43.195 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:43.196 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:43.198 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.198 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.199 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.203 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.203 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.203 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.204 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:43.215 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:43.216 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:43.221 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.221 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.222 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.224 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.224 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.224 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:43.255 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:43.256 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:43.259 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.259 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.260 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.261 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.262 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.262 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.264 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:43.295 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:43.296 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:43.297 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.302 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.304 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:43.335 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:43.337 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:43.338 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.338 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.340 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.342 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.342 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.343 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.344 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:43.475 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:43.476 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:43.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.479 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.481 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.481 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.482 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.483 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:43.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:43.683831 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:43.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:43.684125 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:43.695 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:43.696 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:43.697 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.700 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.701 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.702 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.702 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.705 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:43.894 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:43.896 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:43.896 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.897 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:43.899 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.900 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.900 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:43.902 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:44.255 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:44.256 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:44.257 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:44.257 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:44.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:44.260 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:44.260 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:44.261 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:44.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:44.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:44.687165 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:44.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:44.687435 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:44.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:44.724555 4162 raft_consensus.cc:482] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:33:44.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:44.725135 4162 raft_consensus.cc:504] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbbc07deb8414afc8aa0535445f6b7a9" member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } } peers { permanent_uuid: "06f5921fd8e344ffac65d431b5adad19" member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } } peers { permanent_uuid: "e68a8fd751ef475db7d3cb36d7a1667a" member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } }
19:33:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:44.726954 4162 leader_election.cc:288] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cbbc07deb8414afc8aa0535445f6b7a9 (127.3.141.190:35831), 06f5921fd8e344ffac65d431b5adad19 (127.3.141.189:40587)
19:33:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:44.734099 4091 leader_election.cc:334] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 06f5921fd8e344ffac65d431b5adad19 (127.3.141.189:40587): Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:33:44.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:44.734575 4088 leader_election.cc:334] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer cbbc07deb8414afc8aa0535445f6b7a9 (127.3.141.190:35831): Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111)
19:33:44.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:44.734894 4088 leader_election.cc:302] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e68a8fd751ef475db7d3cb36d7a1667a; no voters: 06f5921fd8e344ffac65d431b5adad19, cbbc07deb8414afc8aa0535445f6b7a9
19:33:44.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:44.735388 4162 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:33:45.335 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:45.336 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:45.337 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:45.337 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:45.337 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:45.339 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:45.339 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:45.340 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:45.342 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:45.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:45.690289 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:45.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:45.690588 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:46.693450 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:46.693698 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:47.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:47.278445 4172 raft_consensus.cc:482] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:33:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:47.278796 4172 raft_consensus.cc:504] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbbc07deb8414afc8aa0535445f6b7a9" member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } } peers { permanent_uuid: "06f5921fd8e344ffac65d431b5adad19" member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } } peers { permanent_uuid: "e68a8fd751ef475db7d3cb36d7a1667a" member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } }
19:33:47.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:47.280546 4172 leader_election.cc:288] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cbbc07deb8414afc8aa0535445f6b7a9 (127.3.141.190:35831), 06f5921fd8e344ffac65d431b5adad19 (127.3.141.189:40587)
19:33:47.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:47.285475 4088 leader_election.cc:334] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer cbbc07deb8414afc8aa0535445f6b7a9 (127.3.141.190:35831): Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111)
19:33:47.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:47.285847 4091 leader_election.cc:334] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 06f5921fd8e344ffac65d431b5adad19 (127.3.141.189:40587): Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:33:47.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:47.286154 4091 leader_election.cc:302] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e68a8fd751ef475db7d3cb36d7a1667a; no voters: 06f5921fd8e344ffac65d431b5adad19, cbbc07deb8414afc8aa0535445f6b7a9
19:33:47.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:47.286685 4172 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:33:47.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:47.696466 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:47.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:47.696740 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:47.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:47.737664 4069 master_runner.cc:186] Error getting master registration for 127.3.141.190:35831: Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111),
19:33:47.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:47.738121 4069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:33:47.855 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:47.856 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:47.857 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:47.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:47.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:47.859 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:47.860 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:47.860 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:47.862 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:48.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:48.699716 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:48.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:48.699963 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:49.702986 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:49.703266 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:49.791342 4178 raft_consensus.cc:482] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:33:49.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:49.791750 4178 raft_consensus.cc:504] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbbc07deb8414afc8aa0535445f6b7a9" member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } } peers { permanent_uuid: "06f5921fd8e344ffac65d431b5adad19" member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } } peers { permanent_uuid: "e68a8fd751ef475db7d3cb36d7a1667a" member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } }
19:33:49.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:49.793419 4178 leader_election.cc:288] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cbbc07deb8414afc8aa0535445f6b7a9 (127.3.141.190:35831), 06f5921fd8e344ffac65d431b5adad19 (127.3.141.189:40587)
19:33:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:49.795332 4088 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111) [suppressed 5 similar messages]
19:33:49.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:49.798588 4088 leader_election.cc:334] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer cbbc07deb8414afc8aa0535445f6b7a9 (127.3.141.190:35831): Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111)
19:33:49.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:49.799026 4091 leader_election.cc:334] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 06f5921fd8e344ffac65d431b5adad19 (127.3.141.189:40587): Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:33:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:49.799609 4091 leader_election.cc:302] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e68a8fd751ef475db7d3cb36d7a1667a; no voters: 06f5921fd8e344ffac65d431b5adad19, cbbc07deb8414afc8aa0535445f6b7a9
19:33:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:49.800434 4178 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:33:50.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:50.706106 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:50.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:50.706358 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:50.815 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:50.816 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:50.817 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:50.817 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:50.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:50.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:50.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:50.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:50.821 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:51.195 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:51.197 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:51.197 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:51.197 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:51.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:51.200 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:51.200 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:51.200 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:51.203 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:51.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:51.709458 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:51.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:51.709797 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:52.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:33:52 dist-test-slave-555z krb5kdc[3650](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364832, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:33:52.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:52.361348 4083 init.cc:295] Successfully reacquired a new kerberos TGT
19:33:52.595 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:52.596 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:52.597 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:52.597 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:52.598 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:52.599 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:52.599 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:52.599 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:52.601 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:52.695 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:52.696 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:52.697 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:52.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:52.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:52.699 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:52.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:52.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:52.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:52.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:52.712549 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:52.712797 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:52.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:52.741532 4069 master_runner.cc:186] Error getting master registration for 127.3.141.190:35831: Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111),
19:33:52.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:52.742024 4069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:33:53.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:53.484552 3061 debug-util.cc:398] Leaking SignalData structure 0x7b0800076660 after lost signal to thread 2996
19:33:53.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:53.485181 3061 debug-util.cc:398] Leaking SignalData structure 0x7b08000bdd00 after lost signal to thread 3064
19:33:53.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:53.717017 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:53.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:53.717372 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:54.535 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:54.536 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:54.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:54.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:54.539 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:54.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:54.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:54.540 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:54.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:54.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:54.721565 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:54.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:54.721951 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:55.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:55.724768 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:55.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:55.725018 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:56.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:56.727638 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:56.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:56.727871 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:57.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:57.200923 4189 raft_consensus.cc:482] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:33:57.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:57.201479 4189 raft_consensus.cc:504] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbbc07deb8414afc8aa0535445f6b7a9" member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } } peers { permanent_uuid: "06f5921fd8e344ffac65d431b5adad19" member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } } peers { permanent_uuid: "e68a8fd751ef475db7d3cb36d7a1667a" member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } }
19:33:57.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:57.203786 4189 leader_election.cc:288] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cbbc07deb8414afc8aa0535445f6b7a9 (127.3.141.190:35831), 06f5921fd8e344ffac65d431b5adad19 (127.3.141.189:40587)
19:33:57.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:57.205776 4088 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111) [suppressed 1 similar messages]
19:33:57.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:57.208648 4091 leader_election.cc:334] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 06f5921fd8e344ffac65d431b5adad19 (127.3.141.189:40587): Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:33:57.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:57.209187 4088 leader_election.cc:334] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer cbbc07deb8414afc8aa0535445f6b7a9 (127.3.141.190:35831): Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111)
19:33:57.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:57.209573 4088 leader_election.cc:302] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e68a8fd751ef475db7d3cb36d7a1667a; no voters: 06f5921fd8e344ffac65d431b5adad19, cbbc07deb8414afc8aa0535445f6b7a9
19:33:57.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:57.210199 4189 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:33:57.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:57.730510 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:57.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:57.730837 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:57.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:57.744539 4069 master_runner.cc:186] Error getting master registration for 127.3.141.190:35831: Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111),
19:33:57.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:33:57.744865 4069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:33:57.774 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:33:57.776 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:33:57.776 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:57.776 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:57.777 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:33:57.779 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:57.779 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:57.780 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:33:57.781 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:33:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:58.733850 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:58.734169 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:33:59.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:59.736760 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:33:59.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:33:59.737001 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:34:00.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:00.739678 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:34:00.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:00.739933 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:34:01.335 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:34:01.336 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:34:01.337 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:01.337 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:01.337 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:01.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:01.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:01.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:01.342 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:34:01.655 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:34:01.655 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:34:01.656 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:01.656 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:01.656 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:01.657 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:01.657 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:01.657 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:01.659 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:34:01.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:01.743413 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:34:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:01.743798 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:34:02.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 19:34:02 dist-test-slave-555z krb5kdc[3650](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790364842, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.141.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:34:02.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:02.363709 4083 init.cc:295] Successfully reacquired a new kerberos TGT
19:34:02.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:02.747665 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:34:02.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:02.747920 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:34:02.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:02.748438 4069 master_runner.cc:186] Error getting master registration for 127.3.141.190:35831: Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111),
19:34:02.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:02.748839 4069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:34:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:03.750597 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:34:03.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:03.750855 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:34:03.834 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:34:03.836 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:34:03.836 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:03.836 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:03.836 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:03.837 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:03.837 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:03.837 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:03.840 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:34:04.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:04.037295 4200 raft_consensus.cc:482] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:34:04.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:04.038223 4200 raft_consensus.cc:504] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbbc07deb8414afc8aa0535445f6b7a9" member_type: VOTER last_known_addr { host: "127.3.141.190" port: 35831 } } peers { permanent_uuid: "06f5921fd8e344ffac65d431b5adad19" member_type: VOTER last_known_addr { host: "127.3.141.189" port: 40587 } } peers { permanent_uuid: "e68a8fd751ef475db7d3cb36d7a1667a" member_type: VOTER last_known_addr { host: "127.3.141.188" port: 33891 } }
19:34:04.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:04.040575 4200 leader_election.cc:288] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cbbc07deb8414afc8aa0535445f6b7a9 (127.3.141.190:35831), 06f5921fd8e344ffac65d431b5adad19 (127.3.141.189:40587)
19:34:04.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:04.043864 4091 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111) [suppressed 1 similar messages]
19:34:04.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:04.058420 4091 leader_election.cc:334] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 06f5921fd8e344ffac65d431b5adad19 (127.3.141.189:40587): Network error: Client connection negotiation failed: client connection to 127.3.141.189:40587: connect: Connection refused (error 111)
19:34:04.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:04.060071 4088 leader_election.cc:334] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer cbbc07deb8414afc8aa0535445f6b7a9 (127.3.141.190:35831): Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111)
19:34:04.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:04.061523 4088 leader_election.cc:302] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e68a8fd751ef475db7d3cb36d7a1667a; no voters: 06f5921fd8e344ffac65d431b5adad19, cbbc07deb8414afc8aa0535445f6b7a9
19:34:04.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:04.063031 4200 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:34:04.355 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:34:04.356 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:34:04.356 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:04.357 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:04.357 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:04.357 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:04.357 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:04.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:04.359 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:34:04.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:04.753628 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:34:04.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:04.753880 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:34:04.895 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:34:04.895 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:34:04.896 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:04.896 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:04.896 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:04.897 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:04.897 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:04.897 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:04.900 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:34:05.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:05.058971 3141 debug-util.cc:398] Leaking SignalData structure 0x7b080003f500 after lost signal to thread 3072
19:34:05.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:05.059646 3141 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1b60 after lost signal to thread 3144
19:34:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:05.756731 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:34:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:05.756971 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:34:06.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:06.759691 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:34:06.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:06.759941 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:34:07.515 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:34:07.516 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:34:07.516 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:07.516 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:07.517 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:07.518 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:07.518 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:07.518 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:07.520 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:34:07.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:07.751124 4069 master_runner.cc:186] Error getting master registration for 127.3.141.190:35831: Network error: Client connection negotiation failed: client connection to 127.3.141.190:35831: connect: Connection refused (error 111),
19:34:07.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:07.751452 4069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:34:07.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:07.762997 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:34:07.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:07.763334 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:34:08.415 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:34:08.416 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:34:08.416 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:08.416 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:08.417 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:08.418 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:08.418 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:08.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:08.419 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:34:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:08.766129 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:34:08.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:08.766387 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:34:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:09.769053 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:34:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:09.769308 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:34:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:10.771996 4156 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e68a8fd751ef475db7d3cb36d7a1667a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:34:10.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 19:34:10.772244 4156 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:34:10.835 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.189:40587(127.3.141.189:40587)] connecting to peer
19:34:10.835 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.141.190:35831(127.3.141.190:35831)] connecting to peer
19:34:10.836 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:10.836 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.141.189:40587(127.3.141.189:40587)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:10.836 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.189:40587: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587
19:34:10.837 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:10.837 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.190:35831(127.3.141.190:35831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:10.837 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.141.190:35831: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
19:34:10.839 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891; will retry
19:34:10.843 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28326), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [131ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [234ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [234ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [619ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [624ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [643ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [650ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [652ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [653ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [680ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [684ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [685ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [686ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [703ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [706ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [708ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [709ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [741ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [744ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [747ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [780ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [783ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [785ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [817ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [821ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [824ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [826ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [827ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [956ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [960ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [963ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [965ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [966ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1176ms] refreshing cache from master, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1180ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1184ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1186ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1187ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1376ms] refreshing cache from master, [1376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1379ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1382ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1384ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1385ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1736ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1740ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1743ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1744ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1744ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [2816ms] refreshing cache from master, [2817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [2818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [2819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [2819ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [2822ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [2824ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [2824ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [5336ms] refreshing cache from master, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [5339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [5340ms] too many traces: truncated at 100 traces, deferred=Deferred@252471082(state=PENDING, result=null, callback=(continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981) -> (continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752), errback=(continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981) -> (continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752)))
19:34:10.848 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.3.141.190:35831(127.3.141.190:35831)] disconnecting while in state TERMINATED
19:34:10.849 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.3.141.188:33891(127.3.141.188:33891)] disconnecting while in state READY
19:34:10.849 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.3.141.189:40587(127.3.141.189:40587)] disconnecting while in state TERMINATED
19:34:10.853 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:34:10.855 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.141.188:33891(127.3.141.188:33891)] handling channelInactive
19:34:10.855 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.141.188:33891(127.3.141.188:33891)] cleaning up while in state READY due to: connection closed
19:34:10.857 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:34:10.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:10.863415 3638 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu with pid 4069
19:34:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T19:34:12Z chronyd exiting
19:34:13.099 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:34:13.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:13.101245 2910 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu with pid 3190
19:34:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:13.123980 2910 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu with pid 3351
19:34:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:13.152487 2910 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu with pid 3494
19:34:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:13.185138 2910 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu with pid 2922
19:34:13.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:13.215530 2910 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu with pid 2992
19:34:13.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 19:34:13.248900 2910 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskQtGm0f/build/tsan/bin/kudu with pid 3067
19:34:14.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T19:34:14Z chronyd exiting
19:34:15.422 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28326), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [131ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [234ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [234ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [619ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [624ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [643ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [650ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [652ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [653ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [680ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [684ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [685ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [686ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [703ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [706ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [708ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [709ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [741ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [744ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [747ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [780ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [783ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [785ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [817ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [821ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [824ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [826ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [827ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [956ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [960ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [963ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [965ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [966ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1176ms] refreshing cache from master, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1180ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1184ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1186ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1187ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1376ms] refreshing cache from master, [1376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1379ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1382ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1384ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1385ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1736ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1740ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1743ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1744ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1744ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [2816ms] refreshing cache from master, [2817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [2818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [2819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [2819ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [2822ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [2824ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [2824ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [5336ms] refreshing cache from master, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [5339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [5340ms] too many traces: truncated at 100 traces, deferred=Deferred@252471082(state=PENDING, result=null, callback=(continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981) -> (continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752), errback=(continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981) -> (continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.141.190:35831
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28326), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [131ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [234ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [234ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [619ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [624ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [643ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [650ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [652ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [653ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [680ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [684ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [685ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [686ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [703ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [706ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [708ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [709ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [741ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [744ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [747ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [780ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [783ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [785ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [817ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [821ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [824ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [826ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [827ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [956ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [960ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [963ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [965ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [966ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1176ms] refreshing cache from master, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1180ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1184ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1186ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1187ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1376ms] refreshing cache from master, [1376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1379ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1382ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1384ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1385ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1736ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1740ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1743ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1744ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1744ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [2816ms] refreshing cache from master, [2817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [2818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [2819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [2819ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [2822ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [2824ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [2824ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [5336ms] refreshing cache from master, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [5339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [5340ms] too many traces: truncated at 100 traces, deferred=Deferred@252471082(state=PENDING, result=null, callback=(continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981) -> (continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752), errback=(continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981) -> (continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.141.190:35831
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
19:34:16.679 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
19:34:16.679 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28326), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [131ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [234ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [234ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [619ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [624ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [643ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [650ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [652ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [653ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [680ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [684ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [685ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [686ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [703ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [706ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [708ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [709ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [741ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [744ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [747ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [780ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [783ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [785ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [817ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [821ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [824ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [826ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [827ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [956ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [960ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [963ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [965ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [966ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1176ms] refreshing cache from master, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1180ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1184ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1186ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1187ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1376ms] refreshing cache from master, [1376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1379ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1382ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1384ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1385ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1736ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1740ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1743ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1744ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1744ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [2816ms] refreshing cache from master, [2817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [2818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [2819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [2819ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [2822ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [2824ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [2824ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [5336ms] refreshing cache from master, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [5339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [5340ms] too many traces: truncated at 100 traces, deferred=Deferred@252471082(state=PENDING, result=null, callback=(continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981) -> (continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752), errback=(continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981) -> (continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.141.190:35831
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28326), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [131ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [234ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [234ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [619ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [624ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [643ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [650ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [652ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [653ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [680ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [684ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [685ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [686ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [703ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [706ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [708ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [709ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [741ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [744ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [745ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [747ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [780ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [783ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [785ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [786ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [817ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [821ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [824ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [826ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [827ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [956ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [960ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [963ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [965ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [966ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1176ms] refreshing cache from master, [1176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1180ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1184ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1186ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1187ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1376ms] refreshing cache from master, [1376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1379ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1379ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1382ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1384ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1385ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1736ms] refreshing cache from master, [1737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [1738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [1739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [1740ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [1743ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [1744ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [1744ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [2816ms] refreshing cache from master, [2817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [2818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [2819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [2819ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.189:40587: Network error: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587, [2822ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.190:35831: Network error: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [2824ms] Sub RPC ConnectToMaster: received response from server master-127.3.141.188:33891: OK, [2824ms] delaying RPC due to: Service unavailable: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831, [5336ms] refreshing cache from master, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.189:40587, [5337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.190:35831, [5339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.141.188:33891, [5340ms] too many traces: truncated at 100 traces, deferred=Deferred@252471082(state=PENDING, result=null, callback=(continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981) -> (continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752), errback=(continuation of Deferred@151931680 after retry RPC after error@611520581) -> (continuation of Deferred@1419081759 after retry RPC after error@952922256) -> (continuation of Deferred@1138781154 after retry RPC after error@2142087907) -> (continuation of Deferred@483465 after retry RPC after error@1528092905) -> (continuation of Deferred@468391698 after retry RPC after error@159603774) -> (continuation of Deferred@851142319 after retry RPC after error@1335178113) -> (continuation of Deferred@210104038 after retry RPC after error@527704548) -> (continuation of Deferred@1989386939 after retry RPC after error@276598483) -> (continuation of Deferred@867016932 after retry RPC after error@233364596) -> (continuation of Deferred@1711612168 after retry RPC after error@318451981) -> (continuation of Deferred@1276053814 after retry RPC after error@122644415) -> (continuation of Deferred@1208818495 after retry RPC after error@53815402) -> (continuation of Deferred@935839619 after retry RPC after error@1263086683) -> (continuation of Deferred@1799160558 after retry RPC after error@1096876443) -> (continuation of Deferred@1217605099 after retry RPC after error@1245379414) -> (continuation of Deferred@1234418317 after retry RPC after error@11995133) -> (continuation of Deferred@1369083509 after retry RPC after error@1281834843) -> (continuation of Deferred@418689928 after retry RPC after error@549771140) -> (continuation of Deferred@623381711 after retry RPC after error@1346396886) -> (continuation of Deferred@1838543781 after retry RPC after error@1802199396) -> (continuation of Deferred@247486130 after retry RPC after error@1573241710) -> (continuation of Deferred@529468328 after retry RPC after error@803158396) -> (continuation of Deferred@589733875 after retry RPC after error@1342961226) -> (continuation of Deferred@926840232 after retry RPC after error@1635565290) -> (continuation of Deferred@1991404593 after retry RPC after error@2038725899) -> (continuation of Deferred@1037288614 after retry RPC after error@970761752)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.141.189:40587,127.3.141.190:35831,127.3.141.188:33891) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.189:40587(127.3.141.189:40587): Connection refused: /127.3.141.189:40587,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.141.190:35831(127.3.141.190:35831): Connection refused: /127.3.141.190:35831
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.141.190:35831
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)