05:42:05.007 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
05:42:05.020 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task_t1FW1/test-tmp
05:42:05.022 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:42:05.026 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task_t1FW1/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:42:05.307 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
05:42:05.444 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:42:05.445 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:42:05.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:05.449046 30288 mini_oidc.cc:152] Starting JWKS server
05:42:05.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:05.453557 30288 webserver.cc:460] Webserver started at http://127.0.0.1:37785/ using document root <none> and password file <none>
05:42:05.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:05.454200 30288 mini_oidc.cc:189] Starting OIDC Discovery server
05:42:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:05.456691 30288 webserver.cc:460] Webserver started at http://127.0.0.1:44507/ using document root <none> and password file <none>
05:42:05.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T05:42:05Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:42:05.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T05:42:05Z Disabled control of system clock
05:42:05.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:05.507901 30288 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:05.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/wal
05:42:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/data
05:42:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:42:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:42:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:42:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/logs
05:42:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/data/info.pb
05:42:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:42:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:42:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:42:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:42:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:42:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:42:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.148.62:42669
05:42:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.148.62
05:42:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.148.20:39369
05:42:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.148.62:42669,127.29.148.61:35845,127.29.148.60:45199
05:42:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37785/jwks.json with env {}
05:42:05.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:05.819155 30300 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:42:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:05.819737 30300 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:42:05.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:05.820710 30300 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:42:05.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:05.855492 30300 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:42:05.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:05.855818 30300 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:42:05.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:05.856032 30300 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:42:05.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:05.856315 30300 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:42:05.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:05.856590 30300 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:42:05.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:05.856793 30300 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37785/jwks.json
05:42:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:05.895823 30300 master_runner.cc:386] Master server non-default flags:
05:42:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.148.20:39369
05:42:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:05.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:05.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/data
05:42:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/wal
05:42:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.148.62:42669,127.29.148.61:35845,127.29.148.60:45199
05:42:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.148.62:42669
05:42:05.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:05.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:42:05.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:05.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37785/jwks.json
05:42:05.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:05.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:05.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/data/info.pb
05:42:05.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.148.62
05:42:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:42:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:42:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:42:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/logs
05:42:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:42:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:42:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:42:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:42:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:42:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:42:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:42:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:42:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:42:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:05.897167 30300 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:42:05.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:05.898907 30300 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:42:05.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:05.917786 30307 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:07.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:05.918608 30306 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:07.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:05.919719 30309 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:07.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:05.920310 30300 server_base.cc:1034] running on GCE node
05:42:07.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.130954 30300 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:42:07.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.134199 30300 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:42:07.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.135648 30300 hybrid_clock.cc:648] HybridClock initialized: now 1790314927135567 us; error 77 us; skew 500 ppm
05:42:07.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.136456 30300 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:42:07.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.148216 30300 webserver.cc:460] Webserver started at http://127.29.148.62:34215/ using document root <none> and password file <none>
05:42:07.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.149173 30300 fs_manager.cc:359] Metadata directory not provided
05:42:07.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.149356 30300 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:42:07.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.149740 30300 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:42:07.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.154577 30300 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/data/instance:
05:42:07.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d9e0febd7c64a55b26dcc2379db6722"
05:42:07.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:07 on dist-test-slave-fjm6"
05:42:07.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.155768 30300 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/wal/instance:
05:42:07.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d9e0febd7c64a55b26dcc2379db6722"
05:42:07.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:07 on dist-test-slave-fjm6"
05:42:07.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.163708 30300 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
05:42:07.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.169131 30316 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:42:07.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.170252 30300 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
05:42:07.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.170617 30300 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/data,/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/wal
05:42:07.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d9e0febd7c64a55b26dcc2379db6722"
05:42:07.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:07 on dist-test-slave-fjm6"
05:42:07.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.171026 30300 fs_report.cc:383] FS layout report
05:42:07.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:42:07.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/wal
05:42:07.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/wal
05:42:07.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/data/data
05:42:07.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:42:07.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:42:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:42:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:42:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:42:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:42:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:42:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:42:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:42:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:42:07.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:42:07.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.223917 30300 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:42:07.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.225571 30300 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:42:07.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.226125 30300 kserver.cc:163] Server-wide thread pool size limit: 3276
05:42:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.310122 30300 rpc_server.cc:307] RPC server started. Bound to: 127.29.148.62:42669
05:42:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.310174 30367 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.148.62:42669 every 8 connection(s)
05:42:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.313396 30300 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/data/info.pb
05:42:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.319060 30368 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:42:07.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.323292 30288 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu as pid 30300
05:42:07.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.323837 30288 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-0/wal/instance
05:42:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.329098 30288 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/wal
05:42:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/data
05:42:07.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/logs
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/data/info.pb
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:07.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.148.61:35845
05:42:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.148.61
05:42:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.148.20:39369
05:42:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:07.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.148.62:42669,127.29.148.61:35845,127.29.148.60:45199
05:42:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:07.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37785/jwks.json with env {}
05:42:07.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.336583 30368 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } has no permanent_uuid. Determining permanent_uuid...
05:42:07.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.366678 30368 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } has no permanent_uuid. Determining permanent_uuid...
05:42:07.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.368796 30317 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.148.61:35845: connect: Connection refused (error 111)
05:42:07.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.373396 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.61:35845: Network error: Client connection negotiation failed: client connection to 127.29.148.61:35845: connect: Connection refused (error 111)
05:42:07.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.422950 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } attempt: 1
05:42:07.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.427058 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.61:35845: Network error: Client connection negotiation failed: client connection to 127.29.148.61:35845: connect: Connection refused (error 111)
05:42:07.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.495585 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } attempt: 2
05:42:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.499583 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.61:35845: Network error: Client connection negotiation failed: client connection to 127.29.148.61:35845: connect: Connection refused (error 111)
05:42:07.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.591086 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } attempt: 3
05:42:07.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.594858 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.61:35845: Network error: Client connection negotiation failed: client connection to 127.29.148.61:35845: connect: Connection refused (error 111)
05:42:07.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.675166 30370 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:42:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.675900 30370 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:42:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.676438 30370 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:42:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.707795 30370 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:42:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.708086 30370 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:42:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.708273 30370 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:42:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.708458 30370 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:42:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.708631 30370 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:42:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.708760 30370 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37785/jwks.json
05:42:07.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.738255 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } attempt: 4
05:42:07.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.741833 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.61:35845: Network error: Client connection negotiation failed: client connection to 127.29.148.61:35845: connect: Connection refused (error 111)
05:42:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.744863 30370 master_runner.cc:386] Master server non-default flags:
05:42:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.148.20:39369
05:42:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/data
05:42:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/wal
05:42:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.148.62:42669,127.29.148.61:35845,127.29.148.60:45199
05:42:07.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:07.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:07.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:07.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.148.61:35845
05:42:07.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:07.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:42:07.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:07.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37785/jwks.json
05:42:07.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:07.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:07.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/data/info.pb
05:42:07.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.148.61
05:42:07.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:07.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:42:07.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:07.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:42:07.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:42:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/logs
05:42:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:42:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:42:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:42:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:42:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:42:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:42:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:42:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:42:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:42:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.746078 30370 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:42:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.747725 30370 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:42:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:08.041281 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } attempt: 5
05:42:08.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:08.046254 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.61:35845: Network error: Client connection negotiation failed: client connection to 127.29.148.61:35845: connect: Connection refused (error 111)
05:42:08.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:08.593725 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } attempt: 6
05:42:08.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:08.601691 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.61:35845: Network error: Client connection negotiation failed: client connection to 127.29.148.61:35845: connect: Connection refused (error 111)
05:42:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.769116 30379 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.769053 30382 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:07.770936 30370 server_base.cc:1034] running on GCE node
05:42:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:07.769066 30380 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:08.973336 30370 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:42:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:08.976435 30370 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:42:08.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:08.977828 30370 hybrid_clock.cc:648] HybridClock initialized: now 1790314928977757 us; error 81 us; skew 500 ppm
05:42:08.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:08.978894 30370 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:42:08.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:08.985595 30370 webserver.cc:460] Webserver started at http://127.29.148.61:33573/ using document root <none> and password file <none>
05:42:08.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:08.986542 30370 fs_manager.cc:359] Metadata directory not provided
05:42:08.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:08.986718 30370 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:42:08.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:08.987093 30370 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:42:08.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:08.992130 30370 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/data/instance:
05:42:08.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef0a65fa03034dc6a584260db3da8390"
05:42:08.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:08 on dist-test-slave-fjm6"
05:42:08.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:08.993392 30370 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/wal/instance:
05:42:08.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef0a65fa03034dc6a584260db3da8390"
05:42:08.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:08 on dist-test-slave-fjm6"
05:42:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.000655 30370 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
05:42:09.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.006698 30391 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:42:09.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.007911 30370 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
05:42:09.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.008214 30370 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/data,/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/wal
05:42:09.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef0a65fa03034dc6a584260db3da8390"
05:42:09.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:08 on dist-test-slave-fjm6"
05:42:09.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.008540 30370 fs_report.cc:383] FS layout report
05:42:09.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:42:09.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/wal
05:42:09.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/wal
05:42:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/data/data
05:42:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:42:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:42:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:42:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:42:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:42:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:42:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:42:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:42:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:42:09.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:42:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:42:09.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.057498 30370 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:42:09.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.059077 30370 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:42:09.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.059561 30370 kserver.cc:163] Server-wide thread pool size limit: 3276
05:42:09.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.141752 30370 rpc_server.cc:307] RPC server started. Bound to: 127.29.148.61:35845
05:42:09.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.141835 30442 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.148.61:35845 every 8 connection(s)
05:42:09.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.144438 30370 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/data/info.pb
05:42:09.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.146729 30288 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu as pid 30370
05:42:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.147244 30288 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-1/wal/instance
05:42:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.149680 30288 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/wal
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/data
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/logs
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/data/info.pb
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.148.60:45199
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.148.60
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.148.20:39369
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.148.62:42669,127.29.148.61:35845,127.29.148.60:45199
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37785/jwks.json with env {}
05:42:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.150430 30443 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:42:09.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.168268 30443 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } has no permanent_uuid. Determining permanent_uuid...
05:42:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.198364 30443 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } has no permanent_uuid. Determining permanent_uuid...
05:42:09.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.214797 30443 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } has no permanent_uuid. Determining permanent_uuid...
05:42:09.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.217504 30392 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:09.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.221719 30443 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:09.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.271293 30443 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 1
05:42:09.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.275692 30443 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:09.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.344372 30443 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 2
05:42:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.348368 30443 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:09.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.439844 30443 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 3
05:42:09.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.443466 30443 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:09.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.489179 30445 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:42:09.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.489707 30445 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:42:09.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.490162 30445 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:42:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.521363 30445 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:42:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.521646 30445 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:42:09.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.521832 30445 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:42:09.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.522006 30445 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:42:09.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.522279 30445 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:42:09.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.522506 30445 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37785/jwks.json
05:42:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.556725 30445 master_runner.cc:386] Master server non-default flags:
05:42:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.148.20:39369
05:42:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/data
05:42:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/wal
05:42:09.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.148.62:42669,127.29.148.61:35845,127.29.148.60:45199
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.148.60:45199
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37785/jwks.json
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/data/info.pb
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.148.60
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/logs
05:42:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:42:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:42:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:42:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:42:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:42:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:42:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:42:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:42:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:42:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.558019 30445 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:42:09.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.560166 30445 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:42:09.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.584019 30455 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.586906 30443 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 4
05:42:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.591370 30443 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.662173 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } attempt: 7
05:42:09.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.697399 30368 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } has no permanent_uuid. Determining permanent_uuid...
05:42:09.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.707283 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:09.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.765997 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 1
05:42:09.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.774124 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:09.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.855858 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 2
05:42:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.862989 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:09.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.890811 30443 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 5
05:42:09.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.897431 30443 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:09.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:09.948571 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 3
05:42:09.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.955078 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.095676 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 4
05:42:10.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:10.101922 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:10.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.385507 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 5
05:42:10.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:10.391647 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.445001 30443 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 6
05:42:10.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:10.455406 30443 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.585801 30458 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:10.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:09.584841 30456 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:10.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:10.929512 30457 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1343 milliseconds
05:42:10.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.930109 30445 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:42:10.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.932145 30445 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:42:10.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.936380 30445 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:42:10.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.937858 30445 hybrid_clock.cc:648] HybridClock initialized: now 1790314930937798 us; error 108 us; skew 500 ppm
05:42:10.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.939097 30445 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:42:10.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.944252 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 6
05:42:10.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:10.951040 30368 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.148.60:45199: Network error: Client connection negotiation failed: client connection to 127.29.148.60:45199: connect: Connection refused (error 111)
05:42:10.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.954908 30445 webserver.cc:460] Webserver started at http://127.29.148.60:36125/ using document root <none> and password file <none>
05:42:10.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.956388 30445 fs_manager.cc:359] Metadata directory not provided
05:42:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.956969 30445 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:42:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.957707 30445 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:42:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.963186 30445 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/data/instance:
05:42:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0fb87707d37247f0803ac49a7e3c9b66"
05:42:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:10 on dist-test-slave-fjm6"
05:42:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.964526 30445 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/wal/instance:
05:42:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0fb87707d37247f0803ac49a7e3c9b66"
05:42:10.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:10 on dist-test-slave-fjm6"
05:42:10.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.974766 30445 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.010s sys 0.000s
05:42:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.980823 30471 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:42:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.981967 30445 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
05:42:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.982321 30445 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/data,/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/wal
05:42:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0fb87707d37247f0803ac49a7e3c9b66"
05:42:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:10 on dist-test-slave-fjm6"
05:42:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:10.982764 30445 fs_report.cc:383] FS layout report
05:42:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:42:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/wal
05:42:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/wal
05:42:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/data/data
05:42:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:42:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:42:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:42:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:42:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:42:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:42:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:42:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:42:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:42:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:42:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:42:11.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.021090 30445 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:42:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.022843 30445 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:42:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.023378 30445 kserver.cc:163] Server-wide thread pool size limit: 3276
05:42:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.106326 30445 rpc_server.cc:307] RPC server started. Bound to: 127.29.148.60:45199
05:42:11.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.106447 30522 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.148.60:45199 every 8 connection(s)
05:42:11.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.109277 30445 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/data/info.pb
05:42:11.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.114809 30288 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu as pid 30445
05:42:11.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.115173 30523 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:42:11.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.115295 30288 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/master-2/wal/instance
05:42:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.131966 30523 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } has no permanent_uuid. Determining permanent_uuid...
05:42:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.161352 30523 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } has no permanent_uuid. Determining permanent_uuid...
05:42:11.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.174746 30523 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } has no permanent_uuid. Determining permanent_uuid...
05:42:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.192554 30523 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66: Bootstrap starting.
05:42:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.200073 30523 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66: Neither blocks nor log segments found. Creating new log.
05:42:11.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.202191 30523 log.cc:824] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66: Log is configured to *not* fsync() on all Append() calls
05:42:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.207624 30523 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66: No bootstrap required, opened a new log
05:42:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.228921 30523 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } }
05:42:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.229525 30523 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:42:11.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.229723 30523 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0fb87707d37247f0803ac49a7e3c9b66, State: Initialized, Role: FOLLOWER
05:42:11.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.230388 30523 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } }
05:42:11.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.233067 30532 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } } }
05:42:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.233747 30532 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [sys.catalog]: This master's current role is: FOLLOWER
05:42:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.234575 30523 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [sys.catalog]: configured and running, proceeding with master startup.
05:42:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:11.263331 30543 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:11.263698 30543 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:11.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:11.280171 30477 tablet.cc:2369] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:42:11.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.257912 30445 master_runner.cc:186] Error getting master registration for 127.29.148.62:42669: OK, instance_id { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" instance_seqno: 1790314927284896 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:42:11.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.285584 30445 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:42:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.516003 30443 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 7
05:42:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.537395 30443 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390: Bootstrap starting.
05:42:11.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.544952 30443 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390: Neither blocks nor log segments found. Creating new log.
05:42:11.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.547727 30443 log.cc:824] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390: Log is configured to *not* fsync() on all Append() calls
05:42:11.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.553045 30443 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390: No bootstrap required, opened a new log
05:42:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.583544 30443 raft_consensus.cc:348] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } }
05:42:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.584260 30443 raft_consensus.cc:374] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:42:11.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.584506 30443 raft_consensus.cc:729] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef0a65fa03034dc6a584260db3da8390, State: Initialized, Role: FOLLOWER
05:42:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.585462 30443 consensus_queue.cc:260] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } }
05:42:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.588619 30547 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } } }
05:42:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.589617 30547 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [sys.catalog]: This master's current role is: FOLLOWER
05:42:11.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.592419 30443 sys_catalog.cc:564] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [sys.catalog]: configured and running, proceeding with master startup.
05:42:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:11.625945 30558 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:11.627074 30558 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.620663 30370 master_runner.cc:186] Error getting master registration for 127.29.148.62:42669: OK, instance_id { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" instance_seqno: 1790314927284896 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:42:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.649648 30370 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:42:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.726329 30532 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:42:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.726881 30532 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } }
05:42:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.729480 30532 leader_election.cc:288] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1d9e0febd7c64a55b26dcc2379db6722 (127.29.148.62:42669), ef0a65fa03034dc6a584260db3da8390 (127.29.148.61:35845)
05:42:11.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.732012 30418 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0fb87707d37247f0803ac49a7e3c9b66" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef0a65fa03034dc6a584260db3da8390" is_pre_election: true
05:42:11.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.733343 30418 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0fb87707d37247f0803ac49a7e3c9b66 in term 0.
05:42:11.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.735728 30472 leader_election.cc:302] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0fb87707d37247f0803ac49a7e3c9b66, ef0a65fa03034dc6a584260db3da8390; no voters:
05:42:11.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.737782 30532 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [term 0 FOLLOWER]: Leader pre-election won for term 1
05:42:11.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.738124 30532 raft_consensus.cc:482] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
05:42:11.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.738430 30532 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [term 0 FOLLOWER]: Advancing to term 1
05:42:11.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.745417 30532 raft_consensus.cc:504] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } }
05:42:11.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.749140 30532 leader_election.cc:288] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [CANDIDATE]: Term 1 election: Requested vote from peers 1d9e0febd7c64a55b26dcc2379db6722 (127.29.148.62:42669), ef0a65fa03034dc6a584260db3da8390 (127.29.148.61:35845)
05:42:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.750388 30418 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0fb87707d37247f0803ac49a7e3c9b66" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef0a65fa03034dc6a584260db3da8390"
05:42:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.751014 30418 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [term 0 FOLLOWER]: Advancing to term 1
05:42:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.759384 30418 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0fb87707d37247f0803ac49a7e3c9b66 in term 1.
05:42:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.760622 30472 leader_election.cc:302] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0fb87707d37247f0803ac49a7e3c9b66, ef0a65fa03034dc6a584260db3da8390; no voters:
05:42:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.761641 30532 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [term 1 FOLLOWER]: Leader election won for term 1
05:42:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.764503 30532 raft_consensus.cc:686] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [term 1 LEADER]: Becoming Leader. State: Replica: 0fb87707d37247f0803ac49a7e3c9b66, State: Running, Role: LEADER
05:42:11.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.765719 30532 consensus_queue.cc:237] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } }
05:42:11.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.778345 30560 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0fb87707d37247f0803ac49a7e3c9b66. Latest consensus state: current_term: 1 leader_uuid: "0fb87707d37247f0803ac49a7e3c9b66" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } } }
05:42:11.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.779227 30560 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [sys.catalog]: This master's current role is: LEADER
05:42:11.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.790804 30562 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
05:42:11.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.798887 30562 catalog_manager.cc:1484] Initializing Kudu cluster ID...
05:42:11.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:11.818596 30397 tablet.cc:2369] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:42:11.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.825664 30418 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [term 1 FOLLOWER]: Refusing update from remote peer 0fb87707d37247f0803ac49a7e3c9b66: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:42:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.828018 30532 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:42:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.853979 30547 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0fb87707d37247f0803ac49a7e3c9b66. Latest consensus state: current_term: 1 leader_uuid: "0fb87707d37247f0803ac49a7e3c9b66" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } } }
05:42:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.855185 30547 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [sys.catalog]: This master's current role is: FOLLOWER
05:42:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.863150 30563 mvcc.cc:205] Tried to move back new op lower bound from 7333129960721780736 to 7333129960522629120. Current Snapshot: MvccSnapshot[applied={T|T < 7333129960721780736}]
05:42:11.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.868669 30560 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0fb87707d37247f0803ac49a7e3c9b66" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } } }
05:42:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.869474 30560 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [sys.catalog]: This master's current role is: LEADER
05:42:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.875015 30562 catalog_manager.cc:1347] Generated new cluster ID: bf2ea62192474f7f9fc9fb015efa926f
05:42:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.875324 30562 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
05:42:11.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.879324 30547 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0fb87707d37247f0803ac49a7e3c9b66" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } } }
05:42:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.880801 30547 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390 [sys.catalog]: This master's current role is: FOLLOWER
05:42:11.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.917948 30562 catalog_manager.cc:1370] Generated new certificate authority record
05:42:11.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.923233 30562 catalog_manager.cc:1504] Loading token signing keys...
05:42:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.951372 30562 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66: Generated new TSK 0
05:42:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.953420 30562 catalog_manager.cc:1514] Initializing in-progress tserver states...
05:42:11.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.984385 30368 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } attempt: 7
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:11.998576 30288 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/wal
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/data
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/logs
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/data/info.pb
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:42:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.148.1:0
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.148.1
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.148.1
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.148.62:42669,127.29.148.61:35845,127.29.148.60:45199
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.148.20:39369
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:42:12.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.041188 30368 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722: Bootstrap starting.
05:42:12.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.064831 30368 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722: Neither blocks nor log segments found. Creating new log.
05:42:12.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.067674 30368 log.cc:824] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722: Log is configured to *not* fsync() on all Append() calls
05:42:12.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.089236 30368 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722: No bootstrap required, opened a new log
05:42:12.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.360558 30368 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } }
05:42:12.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.369597 30368 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:42:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.373039 30368 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1d9e0febd7c64a55b26dcc2379db6722, State: Initialized, Role: FOLLOWER
05:42:12.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.383319 30368 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } }
05:42:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.412408 30572 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:42:12.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.413686 30572 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } }
05:42:12.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.417439 30572 leader_election.cc:288] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ef0a65fa03034dc6a584260db3da8390 (127.29.148.61:35845), 0fb87707d37247f0803ac49a7e3c9b66 (127.29.148.60:45199)
05:42:12.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.418686 30418 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1d9e0febd7c64a55b26dcc2379db6722" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef0a65fa03034dc6a584260db3da8390" is_pre_election: true
05:42:12.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.415894 30571 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } } }
05:42:12.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.420526 30571 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [sys.catalog]: This master's current role is: FOLLOWER
05:42:12.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.424937 30368 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [sys.catalog]: configured and running, proceeding with master startup.
05:42:12.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.427834 30341 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [term 0 FOLLOWER]: Advancing to term 1
05:42:12.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.431639 30342 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0fb87707d37247f0803ac49a7e3c9b66" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1d9e0febd7c64a55b26dcc2379db6722"
05:42:12.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.433452 30343 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "0fb87707d37247f0803ac49a7e3c9b66" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1d9e0febd7c64a55b26dcc2379db6722" is_pre_election: true
05:42:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.440248 30341 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [term 1 FOLLOWER]: Refusing update from remote peer 0fb87707d37247f0803ac49a7e3c9b66: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:42:12.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.455142 30532 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:42:12.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.452462 30498 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1d9e0febd7c64a55b26dcc2379db6722" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0fb87707d37247f0803ac49a7e3c9b66" is_pre_election: true
05:42:12.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.458069 30317 leader_election.cc:302] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1d9e0febd7c64a55b26dcc2379db6722; no voters: 0fb87707d37247f0803ac49a7e3c9b66, ef0a65fa03034dc6a584260db3da8390
05:42:12.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.458889 30571 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:42:12.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:12.487373 30322 tablet.cc:2369] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:42:12.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.548341 30576 mvcc.cc:205] Tried to move back new op lower bound from 7333129960721780736 to 7333129960522629120. Current Snapshot: MvccSnapshot[applied={T|T < 7333129960721780736}]
05:42:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.591621 30571 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 0fb87707d37247f0803ac49a7e3c9b66. Latest consensus state: current_term: 1 leader_uuid: "0fb87707d37247f0803ac49a7e3c9b66" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } } }
05:42:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.591677 30572 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "0fb87707d37247f0803ac49a7e3c9b66" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } } }
05:42:12.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.593113 30572 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [sys.catalog]: This master's current role is: FOLLOWER
05:42:12.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.592444 30571 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722 [sys.catalog]: This master's current role is: FOLLOWER
05:42:12.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.620796 30588 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
05:42:12.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.633575 30558 catalog_manager.cc:1259] Loaded cluster ID: bf2ea62192474f7f9fc9fb015efa926f
05:42:12.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.634191 30558 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390: loading cluster ID for follower catalog manager: success
05:42:12.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.642712 30558 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390: acquiring CA information for follower catalog manager: success
05:42:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.654116 30558 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ef0a65fa03034dc6a584260db3da8390: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:42:12.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:12.697252 30569 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:42:12.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:12.697961 30569 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:42:12.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:12.700270 30569 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:42:12.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:12.760479 30569 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:42:12.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:12.761870 30569 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.148.1
05:42:12.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.826130 30569 tablet_server_runner.cc:78] Tablet server non-default flags:
05:42:12.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:12.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.148.20:39369
05:42:12.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/data
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/wal
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.148.1:0
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/data/info.pb
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.148.1
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.148.62:42669,127.29.148.61:35845,127.29.148.60:45199
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.148.1
05:42:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/logs
05:42:12.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:12.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:42:12.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:42:12.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:42:12.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:42:12.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:42:12.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:42:12.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:42:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:42:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:42:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.827765 30569 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:42:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.830056 30569 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:42:12.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:12.856711 30594 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.981060 30599 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "0fb87707d37247f0803ac49a7e3c9b66" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1d9e0febd7c64a55b26dcc2379db6722" member_type: VOTER last_known_addr { host: "127.29.148.62" port: 42669 } } peers { permanent_uuid: "ef0a65fa03034dc6a584260db3da8390" member_type: VOTER last_known_addr { host: "127.29.148.61" port: 35845 } } peers { permanent_uuid: "0fb87707d37247f0803ac49a7e3c9b66" member_type: VOTER last_known_addr { host: "127.29.148.60" port: 45199 } } }
05:42:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:12.990315 30599 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0fb87707d37247f0803ac49a7e3c9b66 [sys.catalog]: This master's current role is: LEADER
05:42:13.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:13.642225 30588 catalog_manager.cc:1259] Loaded cluster ID: bf2ea62192474f7f9fc9fb015efa926f
05:42:13.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:13.642560 30588 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722: loading cluster ID for follower catalog manager: success
05:42:13.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:13.673153 30588 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722: acquiring CA information for follower catalog manager: success
05:42:13.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:13.699523 30588 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 1d9e0febd7c64a55b26dcc2379db6722: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:42:15.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:14.249562 30593 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 30569
05:42:15.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:12.858840 30595 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:15.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:15.089389 30569 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 2.235s user 0.833s sys 1.225s
05:42:15.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:15.090473 30569 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 2.236s user 0.834s sys 1.225s
05:42:15.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:15.111689 30604 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.117914 30569 server_base.cc:1034] running on GCE node
05:42:15.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.119403 30569 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:42:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.133205 30569 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:42:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.134721 30569 hybrid_clock.cc:648] HybridClock initialized: now 1790314935134791 us; error 238 us; skew 500 ppm
05:42:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.136188 30569 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:42:15.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.176311 30569 webserver.cc:460] Webserver started at http://127.29.148.1:45455/ using document root <none> and password file <none>
05:42:15.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.178009 30569 fs_manager.cc:359] Metadata directory not provided
05:42:15.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.178587 30569 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:42:15.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.179447 30569 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:42:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.187126 30569 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/data/instance:
05:42:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbb14024c2e64af99e806a4655aeb3b1"
05:42:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:15 on dist-test-slave-fjm6"
05:42:15.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.189178 30569 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/wal/instance:
05:42:15.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbb14024c2e64af99e806a4655aeb3b1"
05:42:15.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:15 on dist-test-slave-fjm6"
05:42:15.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.201005 30569 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.007s sys 0.003s
05:42:15.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.209946 30609 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:42:15.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.211753 30569 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.004s
05:42:15.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.212162 30569 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/data,/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/wal
05:42:15.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbb14024c2e64af99e806a4655aeb3b1"
05:42:15.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:15 on dist-test-slave-fjm6"
05:42:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.212592 30569 fs_report.cc:383] FS layout report
05:42:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:42:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/wal
05:42:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/wal
05:42:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/data/data
05:42:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:42:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:42:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:42:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:42:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:42:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:42:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:42:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:42:15.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:42:15.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:42:15.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:42:15.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.261684 30569 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:42:15.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.263566 30569 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:42:15.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.264088 30569 kserver.cc:163] Server-wide thread pool size limit: 3276
05:42:15.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.267786 30569 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:42:15.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.274142 30569 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:42:15.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.274408 30569 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:42:15.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.274683 30569 ts_tablet_manager.cc:610] Registered 0 tablets
05:42:15.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.274870 30569 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:42:15.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.495325 30569 rpc_server.cc:307] RPC server started. Bound to: 127.29.148.1:45287
05:42:15.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.495428 30722 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.148.1:45287 every 8 connection(s)
05:42:15.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.497993 30569 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/data/info.pb
05:42:15.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.505183 30288 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu as pid 30569
05:42:15.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.505898 30288 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-0/wal/instance
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.521615 30288 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/wal
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/data
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/logs
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/data/info.pb
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.148.2:0
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.148.2
05:42:15.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.148.2
05:42:15.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:15.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.148.62:42669,127.29.148.61:35845,127.29.148.60:45199
05:42:15.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.148.20:39369
05:42:15.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:15.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:15.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:42:15.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.581828 30723 heartbeater.cc:347] Connected to a master server at 127.29.148.60:45199
05:42:15.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.582319 30723 heartbeater.cc:464] Registering TS with master...
05:42:15.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.583513 30723 heartbeater.cc:511] Master 127.29.148.60:45199 requested a full tablet report, sending...
05:42:15.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.587195 30488 ts_manager.cc:194] Registered new tserver with Master: cbb14024c2e64af99e806a4655aeb3b1 (127.29.148.1:45287)
05:42:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.590384 30488 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.148.1:41345
05:42:15.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.595150 30724 heartbeater.cc:347] Connected to a master server at 127.29.148.61:35845
05:42:15.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.595566 30724 heartbeater.cc:464] Registering TS with master...
05:42:15.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.596349 30724 heartbeater.cc:511] Master 127.29.148.61:35845 requested a full tablet report, sending...
05:42:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.599732 30408 ts_manager.cc:194] Registered new tserver with Master: cbb14024c2e64af99e806a4655aeb3b1 (127.29.148.1:45287)
05:42:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.604637 30728 heartbeater.cc:347] Connected to a master server at 127.29.148.62:42669
05:42:15.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.605031 30728 heartbeater.cc:464] Registering TS with master...
05:42:15.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.605764 30728 heartbeater.cc:511] Master 127.29.148.62:42669 requested a full tablet report, sending...
05:42:15.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.609023 30333 ts_manager.cc:194] Registered new tserver with Master: cbb14024c2e64af99e806a4655aeb3b1 (127.29.148.1:45287)
05:42:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:15.917824 30727 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:42:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:15.918388 30727 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:42:15.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:15.919010 30727 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:42:15.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:15.951953 30727 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:42:15.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:15.952915 30727 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.148.2
05:42:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.987974 30727 tablet_server_runner.cc:78] Tablet server non-default flags:
05:42:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.148.20:39369
05:42:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/data
05:42:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/wal
05:42:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.148.2:0
05:42:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:42:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/data/info.pb
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.148.2
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.148.62:42669,127.29.148.61:35845,127.29.148.60:45199
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.148.2
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/logs
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:42:15.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.989526 30727 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:42:15.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:15.991416 30727 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:42:16.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:16.012957 30741 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:16.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:16.597859 30723 heartbeater.cc:503] Master 127.29.148.60:45199 was elected leader, sending a full tablet report...
05:42:17.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:16.013011 30743 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:17.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:16.014813 30740 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:17.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:17.288139 30742 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:42:17.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.288231 30727 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:42:17.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.292062 30727 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:42:17.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.294992 30727 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:42:17.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.296396 30727 hybrid_clock.cc:648] HybridClock initialized: now 1790314937296388 us; error 70 us; skew 500 ppm
05:42:17.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.297245 30727 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:42:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.304631 30727 webserver.cc:460] Webserver started at http://127.29.148.2:32775/ using document root <none> and password file <none>
05:42:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.305831 30727 fs_manager.cc:359] Metadata directory not provided
05:42:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.306094 30727 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:42:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.306641 30727 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:42:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.311815 30727 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/data/instance:
05:42:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4ee43002d5d4b08a3cf139ed7c49e05"
05:42:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:17 on dist-test-slave-fjm6"
05:42:17.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.313001 30727 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/wal/instance:
05:42:17.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4ee43002d5d4b08a3cf139ed7c49e05"
05:42:17.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:17 on dist-test-slave-fjm6"
05:42:17.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.321455 30727 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.003s sys 0.006s
05:42:17.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.328717 30750 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:42:17.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.329924 30727 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
05:42:17.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.330207 30727 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/data,/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/wal
05:42:17.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4ee43002d5d4b08a3cf139ed7c49e05"
05:42:17.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:17 on dist-test-slave-fjm6"
05:42:17.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.330508 30727 fs_report.cc:383] FS layout report
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/wal
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/wal
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/data/data
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:42:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:42:17.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.370461 30727 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:42:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.372100 30727 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:42:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.372597 30727 kserver.cc:163] Server-wide thread pool size limit: 3276
05:42:17.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.375126 30727 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:42:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.380041 30727 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:42:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.380251 30727 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:42:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.380438 30727 ts_tablet_manager.cc:610] Registered 0 tablets
05:42:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.380604 30727 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:42:17.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.555989 30727 rpc_server.cc:307] RPC server started. Bound to: 127.29.148.2:46795
05:42:17.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.556087 30862 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.148.2:46795 every 8 connection(s)
05:42:17.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.559872 30727 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/data/info.pb
05:42:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.566095 30288 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu as pid 30727
05:42:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.566601 30288 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-1/wal/instance
05:42:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.589195 30288 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/wal
05:42:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/data
05:42:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/logs
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/data/info.pb
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:42:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:42:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.148.3:0
05:42:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.148.3
05:42:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.148.3
05:42:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.148.62:42669,127.29.148.61:35845,127.29.148.60:45199
05:42:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.148.20:39369
05:42:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:42:17.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.627223 30863 heartbeater.cc:347] Connected to a master server at 127.29.148.60:45199
05:42:17.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.627761 30863 heartbeater.cc:464] Registering TS with master...
05:42:17.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.627965 30865 heartbeater.cc:347] Connected to a master server at 127.29.148.62:42669
05:42:17.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.628286 30865 heartbeater.cc:464] Registering TS with master...
05:42:17.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.629540 30863 heartbeater.cc:511] Master 127.29.148.60:45199 requested a full tablet report, sending...
05:42:17.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.632007 30865 heartbeater.cc:511] Master 127.29.148.62:42669 requested a full tablet report, sending...
05:42:17.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.632864 30488 ts_manager.cc:194] Registered new tserver with Master: b4ee43002d5d4b08a3cf139ed7c49e05 (127.29.148.2:46795)
05:42:17.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.635367 30333 ts_manager.cc:194] Registered new tserver with Master: b4ee43002d5d4b08a3cf139ed7c49e05 (127.29.148.2:46795)
05:42:17.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.635710 30488 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.148.2:56899
05:42:17.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.641752 30864 heartbeater.cc:347] Connected to a master server at 127.29.148.61:35845
05:42:17.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.642056 30864 heartbeater.cc:464] Registering TS with master...
05:42:17.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.642632 30864 heartbeater.cc:511] Master 127.29.148.61:35845 requested a full tablet report, sending...
05:42:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.645382 30408 ts_manager.cc:194] Registered new tserver with Master: b4ee43002d5d4b08a3cf139ed7c49e05 (127.29.148.2:46795)
05:42:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:17.917022 30873 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:42:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:17.917505 30873 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:42:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:17.917930 30873 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:42:17.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:17.949132 30873 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:42:17.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:17.950057 30873 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.148.3
05:42:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.985284 30873 tablet_server_runner.cc:78] Tablet server non-default flags:
05:42:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.148.20:39369
05:42:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/data
05:42:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/wal
05:42:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.148.3:0
05:42:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:42:17.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/data/info.pb
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.148.3
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.148.62:42669,127.29.148.61:35845,127.29.148.60:45199
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.148.3
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/logs
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:42:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.986402 30873 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:42:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:17.988193 30873 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:42:18.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:18.007843 30880 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:18.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:18.640286 30863 heartbeater.cc:503] Master 127.29.148.60:45199 was elected leader, sending a full tablet report...
05:42:19.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:18.009699 30873 server_base.cc:1034] running on GCE node
05:42:19.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:18.007846 30883 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:19.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:18.008602 30881 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.258827 30873 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:42:19.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.261318 30873 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:42:19.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.262682 30873 hybrid_clock.cc:648] HybridClock initialized: now 1790314939262660 us; error 45 us; skew 500 ppm
05:42:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.263535 30873 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:42:19.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.270560 30873 webserver.cc:460] Webserver started at http://127.29.148.3:33233/ using document root <none> and password file <none>
05:42:19.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.271718 30873 fs_manager.cc:359] Metadata directory not provided
05:42:19.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.271986 30873 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:42:19.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.272557 30873 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:42:19.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.277799 30873 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/data/instance:
05:42:19.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b07c493b589c4559bc9846a9b3746716"
05:42:19.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:19 on dist-test-slave-fjm6"
05:42:19.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.279047 30873 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/wal/instance:
05:42:19.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b07c493b589c4559bc9846a9b3746716"
05:42:19.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:19 on dist-test-slave-fjm6"
05:42:19.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.287251 30873 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
05:42:19.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.293288 30890 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:42:19.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.294564 30873 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
05:42:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.294884 30873 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/data,/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/wal
05:42:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b07c493b589c4559bc9846a9b3746716"
05:42:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:19 on dist-test-slave-fjm6"
05:42:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.295217 30873 fs_report.cc:383] FS layout report
05:42:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:42:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/wal
05:42:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/wal
05:42:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/data/data
05:42:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:42:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:42:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:42:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:42:19.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:42:19.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:42:19.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:42:19.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:42:19.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:42:19.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:42:19.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:42:19.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.340163 30873 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:42:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.341719 30873 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:42:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.342162 30873 kserver.cc:163] Server-wide thread pool size limit: 3276
05:42:19.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.345280 30873 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:42:19.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.349852 30873 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:42:19.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.350178 30873 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:42:19.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.350432 30873 ts_tablet_manager.cc:610] Registered 0 tablets
05:42:19.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.350574 30873 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:42:19.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.522150 30873 rpc_server.cc:307] RPC server started. Bound to: 127.29.148.3:41045
05:42:19.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.522253 31003 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.29.148.3:41045 every 8 connection(s)
05:42:19.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.524709 30873 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/data/info.pb
05:42:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.531525 30288 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu as pid 30873
05:42:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.532087 30288 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster3614734020152669697/ts-2/wal/instance
05:42:19.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.576403 31005 heartbeater.cc:347] Connected to a master server at 127.29.148.61:35845
05:42:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.576910 31005 heartbeater.cc:464] Registering TS with master...
05:42:19.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.578166 31005 heartbeater.cc:511] Master 127.29.148.61:35845 requested a full tablet report, sending...
05:42:19.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.580775 31006 heartbeater.cc:347] Connected to a master server at 127.29.148.62:42669
05:42:19.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.581110 31006 heartbeater.cc:464] Registering TS with master...
05:42:19.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.581302 31004 heartbeater.cc:347] Connected to a master server at 127.29.148.60:45199
05:42:19.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.581668 31004 heartbeater.cc:464] Registering TS with master...
05:42:19.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.581944 31006 heartbeater.cc:511] Master 127.29.148.62:42669 requested a full tablet report, sending...
05:42:19.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.581959 30408 ts_manager.cc:194] Registered new tserver with Master: b07c493b589c4559bc9846a9b3746716 (127.29.148.3:41045)
05:42:19.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.582420 31004 heartbeater.cc:511] Master 127.29.148.60:45199 requested a full tablet report, sending...
05:42:19.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.584467 30333 ts_manager.cc:194] Registered new tserver with Master: b07c493b589c4559bc9846a9b3746716 (127.29.148.3:41045)
05:42:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.584802 30488 ts_manager.cc:194] Registered new tserver with Master: b07c493b589c4559bc9846a9b3746716 (127.29.148.3:41045)
05:42:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.586683 30488 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.148.3:55171
05:42:19.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:19.599402 30288 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
05:42:19.611 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:42:19.612 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
05:42:19.623 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.29.148.62"
port: 42669
}
bound_http_address {
host: "127.29.148.62"
port: 34215
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.29.148.61"
port: 35845
}
bound_http_address {
host: "127.29.148.61"
port: 33573
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.29.148.60"
port: 45199
}
bound_http_address {
host: "127.29.148.60"
port: 36125
}
}
}
05:42:19.628 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:42:19.631 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.29.148.1"
port: 45287
}
bound_http_address {
host: "127.29.148.1"
port: 45455
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.29.148.2"
port: 46795
}
bound_http_address {
host: "127.29.148.2"
port: 32775
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.29.148.3"
port: 41045
}
bound_http_address {
host: "127.29.148.3"
port: 33233
}
}
}
05:42:19.631 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
05:42:19.825 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
05:42:19.956 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task_t1FW1/test-tmp
05:42:19.957 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:42:19.958 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task_t1FW1/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:42:19.962 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
05:42:20.312 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:42:20.313 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:42:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:20.315786 31015 mini_oidc.cc:152] Starting JWKS server
05:42:20.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:20.319761 31015 webserver.cc:460] Webserver started at http://127.0.0.1:36121/ using document root <none> and password file <none>
05:42:20.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:20.320168 31015 mini_oidc.cc:189] Starting OIDC Discovery server
05:42:20.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:20.322254 31015 webserver.cc:460] Webserver started at http://127.0.0.1:46675/ using document root <none> and password file <none>
05:42:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
05:42:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/principal' for realm 'KRBTEST.COM',
05:42:20.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
05:42:20.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:20 dist-test-slave-fjm6 krb5kdc[31027](info): setting up network...
05:42:20.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
05:42:20.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:20 dist-test-slave-fjm6 krb5kdc[31027](info): set up 2 sockets
05:42:20.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:20 dist-test-slave-fjm6 krb5kdc[31027](info): commencing operation
05:42:20.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
05:42:20.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:20.590175 31004 heartbeater.cc:503] Master 127.29.148.60:45199 was elected leader, sending a full tablet report...
05:42:21.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:21.519577 31015 mini_kdc.cc:121] Time spent starting KDC: real 1.172s user 0.000s sys 0.023s
05:42:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
05:42:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:42:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
05:42:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
05:42:21.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:42:21.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
05:42:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
05:42:21.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:42:21.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
05:42:21.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:42:21.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/test-user.keytab.
05:42:21.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/test-user.keytab.
05:42:21.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:21 dist-test-slave-fjm6 krb5kdc[31027](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314941, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:42:21.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:42:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T05:42:21Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:42:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T05:42:21Z Disabled control of system clock
05:42:21.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.73.254@KRBTEST.COM; defaulting to no policy
05:42:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:42:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.73.254@KRBTEST.COM" created.
05:42:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:42:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.73.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.254.keytab.
05:42:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.73.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.254.keytab.
05:42:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:21.720767 31015 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal
05:42:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data
05:42:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:42:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:42:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:42:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/logs
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data/info.pb
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.73.254:33133
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.73.254
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.254.keytab
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.73.254
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.73.212:35393
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.73.254:33133,127.30.73.253:35861,127.30.73.252:36847
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36121/jwks.json
05:42:21.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:42:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:22.070729 31040 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:42:22.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:22.071435 31040 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:42:22.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:22.072402 31040 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:42:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:22.113719 31040 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:42:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:22.114046 31040 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:42:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:22.114477 31040 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:42:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:22.114794 31040 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:42:22.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:22.115105 31040 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:42:22.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:22.115422 31040 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:42:22.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:22.115665 31040 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36121/jwks.json
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:22.167708 31040 master_runner.cc:386] Master server non-default flags:
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.73.212:35393
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.73.254:33133,127.30.73.253:35861,127.30.73.252:36847
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.254.keytab
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.73.254
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.73.254:33133
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36121/jwks.json
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:42:22.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:42:22.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:22.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:22.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data/info.pb
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.73.254
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/logs
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:22.169210 31040 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:42:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:22.170614 31040 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:42:22.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:22.193059 31047 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:23.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:22.193284 31046 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:23.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:22.196734 31049 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:23.590148 31048 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1395 milliseconds
05:42:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.590272 31040 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:42:23.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.591986 31040 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:42:23.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.596887 31040 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:42:23.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.598265 31040 hybrid_clock.cc:648] HybridClock initialized: now 1790314943598206 us; error 86 us; skew 500 ppm
05:42:23.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.599546 31040 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:42:23.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:23 dist-test-slave-fjm6 krb5kdc[31027](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314943, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:42:23.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.612505 31040 init.cc:377] Logged in from keytab as kudu/127.30.73.254@KRBTEST.COM (short username kudu)
05:42:23.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.624923 31040 webserver.cc:460] Webserver started at http://127.30.73.254:35365/ using document root <none> and password file <none>
05:42:23.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.625957 31040 fs_manager.cc:359] Metadata directory not provided
05:42:23.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.626157 31040 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:42:23.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.626654 31040 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:42:23.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.632694 31040 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data/instance:
05:42:23.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "293f1cfbf9d24044ad944ef86b1f760c"
05:42:23.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:23 on dist-test-slave-fjm6"
05:42:23.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.633837 31040 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal/instance:
05:42:23.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "293f1cfbf9d24044ad944ef86b1f760c"
05:42:23.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:23 on dist-test-slave-fjm6"
05:42:23.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.646423 31040 fs_manager.cc:691] Time spent creating directory manager: real 0.012s user 0.005s sys 0.008s
05:42:23.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.658663 31058 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:42:23.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.660452 31040 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.002s sys 0.005s
05:42:23.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.660887 31040 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data,/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal
05:42:23.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "293f1cfbf9d24044ad944ef86b1f760c"
05:42:23.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:23 on dist-test-slave-fjm6"
05:42:23.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.661281 31040 fs_report.cc:383] FS layout report
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data/data
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:42:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:42:23.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.721851 31040 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:42:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.730123 31040 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:42:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.730921 31040 kserver.cc:163] Server-wide thread pool size limit: 3276
05:42:23.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.870712 31040 rpc_server.cc:307] RPC server started. Bound to: 127.30.73.254:33133
05:42:23.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.870849 31109 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.73.254:33133 every 8 connection(s)
05:42:23.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.873621 31040 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data/info.pb
05:42:23.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.876865 31015 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu as pid 31040
05:42:23.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.877444 31015 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal/instance
05:42:23.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.881955 31110 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:42:23.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.73.253@KRBTEST.COM; defaulting to no policy
05:42:23.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.895387 31110 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } has no permanent_uuid. Determining permanent_uuid...
05:42:23.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:42:23.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.73.253@KRBTEST.COM" created.
05:42:23.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:23 dist-test-slave-fjm6 krb5kdc[31027](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314943, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.254@KRBTEST.COM for kudu/127.30.73.254@KRBTEST.COM
05:42:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:42:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.73.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.253.keytab.
05:42:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.73.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.253.keytab.
05:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.927989 31015 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal
05:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data
05:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/logs
05:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data/info.pb
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.73.253:35861
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.73.253
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.253.keytab
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.73.253
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:42:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.73.212:35393
05:42:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.73.254:33133,127.30.73.253:35861,127.30.73.252:36847
05:42:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36121/jwks.json
05:42:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:42:23.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.935380 31110 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } has no permanent_uuid. Determining permanent_uuid...
05:42:23.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:23.937886 31060 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:23.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:23.942919 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.253:35861: Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:23.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:23.992542 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } attempt: 1
05:42:23.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:23.997063 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.253:35861: Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:24.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:24.065476 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } attempt: 2
05:42:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.069923 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.253:35861: Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:24.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:24.161310 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } attempt: 3
05:42:24.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.165263 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.253:35861: Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.269182 31116 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:42:24.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.269709 31116 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:42:24.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.270284 31116 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:42:24.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.301498 31116 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:42:24.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.301723 31116 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:42:24.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.302079 31116 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:42:24.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.302342 31116 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:42:24.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.302542 31116 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:42:24.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.302810 31116 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:42:24.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.303042 31116 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36121/jwks.json
05:42:24.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:24.308696 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } attempt: 4
05:42:24.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.312871 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.253:35861: Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:24.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:24.356491 31116 master_runner.cc:386] Master server non-default flags:
05:42:24.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.73.212:35393
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.73.254:33133,127.30.73.253:35861,127.30.73.252:36847
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.253.keytab
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.73.253
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.73.253:35861
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36121/jwks.json
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data/info.pb
05:42:24.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.73.253
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/logs
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:42:24.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:42:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:42:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:42:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:24.357646 31116 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:42:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:24.358803 31116 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:42:24.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.379052 31126 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:24.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:24.612334 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } attempt: 5
05:42:24.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.618029 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.253:35861: Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:25.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.165447 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } attempt: 6
05:42:25.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:25.172154 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.253:35861: Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.379089 31123 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:25.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:24.379048 31124 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:25.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:25.658179 31125 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:42:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.658685 31116 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:42:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.663087 31116 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:42:25.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.666610 31116 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:42:25.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.667961 31116 hybrid_clock.cc:648] HybridClock initialized: now 1790314945667934 us; error 54 us; skew 500 ppm
05:42:25.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.668794 31116 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:42:25.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:25 dist-test-slave-fjm6 krb5kdc[31027](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314945, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:42:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.679790 31116 init.cc:377] Logged in from keytab as kudu/127.30.73.253@KRBTEST.COM (short username kudu)
05:42:25.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.685797 31116 webserver.cc:460] Webserver started at http://127.30.73.253:33235/ using document root <none> and password file <none>
05:42:25.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.686697 31116 fs_manager.cc:359] Metadata directory not provided
05:42:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.686887 31116 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:42:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.687327 31116 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:42:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.692623 31116 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data/instance:
05:42:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62f2f856c0b749c8bea4bf839de07509"
05:42:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:25 on dist-test-slave-fjm6"
05:42:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.693825 31116 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal/instance:
05:42:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62f2f856c0b749c8bea4bf839de07509"
05:42:25.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:25 on dist-test-slave-fjm6"
05:42:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.702144 31116 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
05:42:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.708757 31137 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:42:25.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.709985 31116 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
05:42:25.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.710278 31116 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data,/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal
05:42:25.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62f2f856c0b749c8bea4bf839de07509"
05:42:25.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:25 on dist-test-slave-fjm6"
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.710608 31116 fs_report.cc:383] FS layout report
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data/data
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:42:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:42:25.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.777400 31116 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:42:25.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.783565 31116 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:42:25.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.783990 31116 kserver.cc:163] Server-wide thread pool size limit: 3276
05:42:25.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.880903 31116 rpc_server.cc:307] RPC server started. Bound to: 127.30.73.253:35861
05:42:25.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.880944 31189 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.73.253:35861 every 8 connection(s)
05:42:25.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.883821 31116 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data/info.pb
05:42:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.887312 31015 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu as pid 31116
05:42:25.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.887764 31015 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal/instance
05:42:25.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.889899 31190 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:42:25.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.73.252@KRBTEST.COM; defaulting to no policy
05:42:25.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.906298 31190 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } has no permanent_uuid. Determining permanent_uuid...
05:42:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:42:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.73.252@KRBTEST.COM" created.
05:42:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:25 dist-test-slave-fjm6 krb5kdc[31027](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314945, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.253@KRBTEST.COM for kudu/127.30.73.254@KRBTEST.COM
05:42:25.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:25.976636 31190 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } has no permanent_uuid. Determining permanent_uuid...
05:42:26.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:26 dist-test-slave-fjm6 krb5kdc[31027](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314945, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.253@KRBTEST.COM for kudu/127.30.73.253@KRBTEST.COM
05:42:26.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.211174 31190 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } has no permanent_uuid. Determining permanent_uuid...
05:42:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.213493 31138 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.216717 31190 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.232553 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } attempt: 7
05:42:26.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:26 dist-test-slave-fjm6 krb5kdc[31027](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314943, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.254@KRBTEST.COM for kudu/127.30.73.253@KRBTEST.COM
05:42:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.266170 31190 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 1
05:42:26.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.281742 31110 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } has no permanent_uuid. Determining permanent_uuid...
05:42:26.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.288661 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.298146 31190 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:26.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.347096 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 1
05:42:26.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.351099 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.367024 31190 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 2
05:42:26.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.371750 31190 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:26.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.432595 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 2
05:42:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.436308 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.463140 31190 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 3
05:42:26.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.467002 31190 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:26.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.521747 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 3
05:42:26.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.525553 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:26.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.610363 31190 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 4
05:42:26.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.614240 31190 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:26.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.665963 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 4
05:42:26.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.669597 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:26.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:42:26.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.73.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.252.keytab.
05:42:26.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.73.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.252.keytab.
05:42:26.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.912195 31015 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.30.73.252: real 1.023s user 0.002s sys 0.012s
05:42:26.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.913645 31190 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 5
05:42:26.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.913769 31015 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:26.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:26.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/wal
05:42:26.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data
05:42:26.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:42:26.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:42:26.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:42:26.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:26.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/logs
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data/info.pb
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.73.252:36847
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.73.252
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.252.keytab
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.73.252
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.73.212:35393
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:26.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:26.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.73.254:33133,127.30.73.253:35861,127.30.73.252:36847
05:42:26.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:26.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36121/jwks.json
05:42:26.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:42:26.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.918211 31190 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:26.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:26.952982 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 5
05:42:26.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:26.956820 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.249647 31201 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:42:27.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.250360 31201 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:42:27.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.251067 31201 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:42:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.283648 31201 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:42:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.283874 31201 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:42:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.284235 31201 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:42:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.284454 31201 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:42:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.284641 31201 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:42:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.284819 31201 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:42:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.284964 31201 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36121/jwks.json
05:42:27.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.290943 31186 debug-util.cc:398] Leaking SignalData structure 0x7b0800073060 after lost signal to thread 31118
05:42:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.291669 31186 debug-util.cc:398] Leaking SignalData structure 0x7b08000737c0 after lost signal to thread 31189
05:42:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:27.342334 31201 master_runner.cc:386] Master server non-default flags:
05:42:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.73.212:35393
05:42:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data
05:42:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/wal
05:42:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:42:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.73.254:33133,127.30.73.253:35861,127.30.73.252:36847
05:42:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.252.keytab
05:42:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.73.252
05:42:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.73.252:36847
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36121/jwks.json
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data/info.pb
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.73.252
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/logs
05:42:27.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:42:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:42:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:42:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:42:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:42:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:42:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:42:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:42:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:42:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:27.343961 31201 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:42:27.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:27.345418 31201 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:42:27.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.370847 31208 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:27.465641 31190 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 6
05:42:27.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.475873 31190 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:27.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:27.509244 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 6
05:42:27.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.518720 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:28.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.536337 31190 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 7
05:42:28.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:28.546300 31190 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:28.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.552284 31110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } attempt: 7
05:42:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:28.577944 31110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.30.73.252:36847: Network error: Client connection negotiation failed: client connection to 127.30.73.252:36847: connect: Connection refused (error 111)
05:42:28.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:27.375392 31201 server_base.cc:1034] running on GCE node
05:42:28.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.372022 31207 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:28.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:27.370968 31210 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:28.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.781621 31201 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:42:28.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.784760 31201 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:42:28.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.786187 31201 hybrid_clock.cc:648] HybridClock initialized: now 1790314948786148 us; error 59 us; skew 500 ppm
05:42:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.786983 31201 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:42:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:28 dist-test-slave-fjm6 krb5kdc[31027](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314948, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:42:28.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.794307 31201 init.cc:377] Logged in from keytab as kudu/127.30.73.252@KRBTEST.COM (short username kudu)
05:42:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.806810 31201 webserver.cc:460] Webserver started at http://127.30.73.252:46477/ using document root <none> and password file <none>
05:42:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.807768 31201 fs_manager.cc:359] Metadata directory not provided
05:42:28.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.807958 31201 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:42:28.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.808346 31201 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:42:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.813441 31201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data/instance:
05:42:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17be7a6354c946de8fc6484884d0dfef"
05:42:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:28 on dist-test-slave-fjm6"
05:42:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.814698 31201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/wal/instance:
05:42:28.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17be7a6354c946de8fc6484884d0dfef"
05:42:28.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:28 on dist-test-slave-fjm6"
05:42:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.823980 31201 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.006s sys 0.001s
05:42:28.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.832104 31229 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:42:28.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.833366 31201 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
05:42:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.833731 31201 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data,/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/wal
05:42:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17be7a6354c946de8fc6484884d0dfef"
05:42:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:28 on dist-test-slave-fjm6"
05:42:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.834095 31201 fs_report.cc:383] FS layout report
05:42:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:42:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/wal
05:42:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/wal
05:42:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data/data
05:42:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:42:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:42:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:42:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:42:28.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:42:28.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:42:28.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:42:28.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:42:28.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:42:28.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:42:28.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:42:28.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.903573 31201 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:42:28.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.910336 31201 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:42:28.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:28.910857 31201 kserver.cc:163] Server-wide thread pool size limit: 3276
05:42:29.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.018378 31201 rpc_server.cc:307] RPC server started. Bound to: 127.30.73.252:36847
05:42:29.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.018697 31280 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.73.252:36847 every 8 connection(s)
05:42:29.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.022862 31201 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data/info.pb
05:42:29.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.030452 31015 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu as pid 31201
05:42:29.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.031000 31015 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/wal/instance
05:42:29.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.039578 31281 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:42:29.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.064311 31281 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } has no permanent_uuid. Determining permanent_uuid...
05:42:29.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:29 dist-test-slave-fjm6 krb5kdc[31027](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314948, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.252@KRBTEST.COM for kudu/127.30.73.254@KRBTEST.COM
05:42:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:29 dist-test-slave-fjm6 krb5kdc[31027](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314941, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.73.252@KRBTEST.COM
05:42:29.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.140488 31281 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } has no permanent_uuid. Determining permanent_uuid...
05:42:29.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:29 dist-test-slave-fjm6 krb5kdc[31027](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314948, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.252@KRBTEST.COM for kudu/127.30.73.253@KRBTEST.COM
05:42:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:29 dist-test-slave-fjm6 krb5kdc[31027](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314941, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.73.253@KRBTEST.COM
05:42:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.215080 31281 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } has no permanent_uuid. Determining permanent_uuid...
05:42:29.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:29 dist-test-slave-fjm6 krb5kdc[31027](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314948, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.252@KRBTEST.COM for kudu/127.30.73.252@KRBTEST.COM
05:42:29.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:29 dist-test-slave-fjm6 krb5kdc[31027](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314941, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.73.254@KRBTEST.COM
05:42:29.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.256522 31015 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
05:42:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.257601 31281 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: Bootstrap starting.
05:42:29.264 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:42:29.266 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
05:42:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.268566 31281 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: Neither blocks nor log segments found. Creating new log.
05:42:29.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.274739 31281 log.cc:824] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: Log is configured to *not* fsync() on all Append() calls
05:42:29.279 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
05:42:29.280 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
05:42:29.284 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.30.73.254"
port: 33133
}
bound_http_address {
host: "127.30.73.254"
port: 35365
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.30.73.253"
port: 35861
}
bound_http_address {
host: "127.30.73.253"
port: 33235
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.30.73.252"
port: 36847
}
bound_http_address {
host: "127.30.73.252"
port: 46477
}
}
}
05:42:29.284 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:42:29.284 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:42:29.285 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:42:29.287 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:42:29.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.296118 31281 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: No bootstrap required, opened a new log
05:42:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.334723 31281 raft_consensus.cc:348] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } } peers { permanent_uuid: "62f2f856c0b749c8bea4bf839de07509" member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } } peers { permanent_uuid: "17be7a6354c946de8fc6484884d0dfef" member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } }
05:42:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.342739 31281 raft_consensus.cc:374] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:42:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.343086 31281 raft_consensus.cc:729] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 17be7a6354c946de8fc6484884d0dfef, State: Initialized, Role: FOLLOWER
05:42:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:29 dist-test-slave-fjm6 krb5kdc[31027](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314949, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:42:29.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:42:29.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.344141 31281 consensus_queue.cc:260] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } } peers { permanent_uuid: "62f2f856c0b749c8bea4bf839de07509" member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } } peers { permanent_uuid: "17be7a6354c946de8fc6484884d0dfef" member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } }
05:42:29.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.355135 31295 sys_catalog.cc:455] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } } peers { permanent_uuid: "62f2f856c0b749c8bea4bf839de07509" member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } } peers { permanent_uuid: "17be7a6354c946de8fc6484884d0dfef" member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } } }
05:42:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.357352 31281 sys_catalog.cc:564] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [sys.catalog]: configured and running, proceeding with master startup.
05:42:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.358419 31295 sys_catalog.cc:458] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [sys.catalog]: This master's current role is: FOLLOWER
05:42:29.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:29.393054 31306 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:29.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:29.393841 31306 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:29.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.387147 31201 master_runner.cc:186] Error getting master registration for 127.30.73.254:33133: OK, instance_id { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" instance_seqno: 1790314943830691 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:42:29.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:29.418391 31235 tablet.cc:2369] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:42:29.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.418902 31201 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:42:29.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:29.442770 31015 mini_kdc.cc:317] Time spent kinit for test-admin: real 0.155s user 0.004s sys 0.011s
05:42:29.443 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:42:29.505 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5cc
05:42:29.515 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:42:29.516 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5cc
05:42:29.518 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:42:29.522 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
05:42:29.524 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:42:29.524 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:42:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:29 dist-test-slave-fjm6 krb5kdc[31027](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314949, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:42:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:42:29.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.571094 31295 raft_consensus.cc:482] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:42:29.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.571723 31295 raft_consensus.cc:504] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } } peers { permanent_uuid: "62f2f856c0b749c8bea4bf839de07509" member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } } peers { permanent_uuid: "17be7a6354c946de8fc6484884d0dfef" member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } }
05:42:29.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.573901 31295 leader_election.cc:288] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 293f1cfbf9d24044ad944ef86b1f760c (127.30.73.254:33133), 62f2f856c0b749c8bea4bf839de07509 (127.30.73.253:35861)
05:42:29.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:29.639506 31015 mini_kdc.cc:317] Time spent kinit for test-admin: real 0.114s user 0.002s sys 0.011s
05:42:29.640 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:42:29.640 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5cc
05:42:29.642 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:42:29.642 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.73.254:33133
05:42:29.644 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
05:42:29.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.645424 31015 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu with pid 31040
05:42:29.669 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:42:29.670 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.73.252:36847
05:42:29.670 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
05:42:29.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.671324 31015 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu with pid 31201
05:42:29.700 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:42:29.701 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.73.253:35861
05:42:29.701 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
05:42:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.701776 31015 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu with pid 31116
05:42:29.729 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:42:29.729 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.73.254:33133
05:42:29.732 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:29.733940 31015 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/logs
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data/info.pb
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:42:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.73.254:33133
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.73.254
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35365
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.254.keytab
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.73.254
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.73.212:35393
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.73.254:33133,127.30.73.253:35861,127.30.73.252:36847
05:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:29.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36121/jwks.json
05:42:29.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:42:30.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:30.048576 31311 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:42:30.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:30.049268 31311 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:42:30.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:30.049863 31311 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:42:30.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:30.081168 31311 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:42:30.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:30.081393 31311 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:42:30.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:30.081642 31311 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:42:30.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:30.081835 31311 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:42:30.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:30.082031 31311 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:42:30.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:30.082218 31311 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:42:30.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:30.082358 31311 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36121/jwks.json
05:42:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:30.120241 31311 master_runner.cc:386] Master server non-default flags:
05:42:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.73.212:35393
05:42:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data
05:42:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal
05:42:30.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:42:30.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:30.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.73.254:33133,127.30.73.253:35861,127.30.73.252:36847
05:42:30.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.254.keytab
05:42:30.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.73.254
05:42:30.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.73.254:33133
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36121/jwks.json
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data/info.pb
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.73.254
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35365
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/logs
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:42:30.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:42:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:42:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:30.121371 31311 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:42:30.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:30.122395 31311 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:42:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:30.142146 31319 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:30.142828 31321 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:30.143771 31311 server_base.cc:1034] running on GCE node
05:42:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:30.144532 31318 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:31.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.389351 31311 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:42:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.392235 31311 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:42:31.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.393631 31311 hybrid_clock.cc:648] HybridClock initialized: now 1790314951393570 us; error 56 us; skew 500 ppm
05:42:31.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.394491 31311 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:42:31.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:31 dist-test-slave-fjm6 krb5kdc[31027](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314951, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:42:31.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.400743 31311 init.cc:377] Logged in from keytab as kudu/127.30.73.254@KRBTEST.COM (short username kudu)
05:42:31.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.406435 31311 webserver.cc:460] Webserver started at http://127.30.73.254:35365/ using document root <none> and password file <none>
05:42:31.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.407626 31311 fs_manager.cc:359] Metadata directory not provided
05:42:31.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.407835 31311 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:42:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.415904 31311 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
05:42:31.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.420746 31330 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:42:31.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.421912 31311 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
05:42:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.422204 31311 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data,/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal
05:42:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "293f1cfbf9d24044ad944ef86b1f760c"
05:42:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:23 on dist-test-slave-fjm6"
05:42:31.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.424645 31311 fs_report.cc:383] FS layout report
05:42:31.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data/data
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:42:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:42:31.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.464483 31311 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:42:31.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.470574 31311 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:42:31.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.471118 31311 kserver.cc:163] Server-wide thread pool size limit: 3276
05:42:31.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.553001 31311 rpc_server.cc:307] RPC server started. Bound to: 127.30.73.254:33133
05:42:31.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.553128 31381 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.73.254:33133 every 8 connection(s)
05:42:31.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.555931 31311 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/data/info.pb
05:42:31.560 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:42:31.560 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.73.252:36847
05:42:31.560 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
05:42:31.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.558904 31015 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu as pid 31311
05:42:31.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.563088 31015 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:31.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:31.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/wal
05:42:31.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data
05:42:31.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:42:31.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:42:31.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:42:31.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:31.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:31.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/logs
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data/info.pb
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.73.252:36847
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.73.252
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46477
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.252.keytab
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.73.252
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:42:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:42:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:42:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.73.212:35393
05:42:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.73.254:33133,127.30.73.253:35861,127.30.73.252:36847
05:42:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36121/jwks.json
05:42:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:42:31.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.577051 31382 sys_catalog.cc:263] Verifying existing consensus state
05:42:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260925 05:42:31.578044 31382 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:42:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.578554 31311 master.cc:537] Master@127.30.73.254:33133 shutting down...
05:42:31.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.634112 31311 master.cc:552] Master@127.30.73.254:33133 shutdown complete.
05:42:31.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:42:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:31.896368 31384 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:42:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:31.897060 31384 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:42:31.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:31.897814 31384 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:42:31.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:31.930526 31384 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:42:31.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:31.930820 31384 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:42:31.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:31.931218 31384 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:42:31.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:31.931596 31384 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:42:31.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:31.931910 31384 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:42:31.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:31.932217 31384 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:42:31.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:31.932443 31384 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36121/jwks.json
05:42:31.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.969314 31384 master_runner.cc:386] Master server non-default flags:
05:42:31.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:31.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.73.212:35393
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/wal
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.73.254:33133,127.30.73.253:35861,127.30.73.252:36847
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.252.keytab
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.73.252
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.73.252:36847
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36121/jwks.json
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data/info.pb
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.73.252
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46477
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:42:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:42:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/logs
05:42:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:42:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:42:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:42:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:42:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:42:31.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:42:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:42:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:42:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:42:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.970827 31384 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:42:31.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.972199 31384 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:42:31.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:31.990595 31390 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:33.209908 30519 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4ee0 after lost signal to thread 30450
05:42:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:33.210629 30519 debug-util.cc:398] Leaking SignalData structure 0x7b08000be000 after lost signal to thread 30522
05:42:33.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:31.993372 31384 server_base.cc:1034] running on GCE node
05:42:33.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:31.990911 31391 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:33.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:31.990607 31393 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:33.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.272378 31384 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:42:33.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.276239 31384 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:42:33.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.277729 31384 hybrid_clock.cc:648] HybridClock initialized: now 1790314953277677 us; error 51 us; skew 500 ppm
05:42:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.278920 31384 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:42:33.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:33 dist-test-slave-fjm6 krb5kdc[31027](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314953, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:42:33.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.286707 31384 init.cc:377] Logged in from keytab as kudu/127.30.73.252@KRBTEST.COM (short username kudu)
05:42:33.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.293481 31384 webserver.cc:460] Webserver started at http://127.30.73.252:46477/ using document root <none> and password file <none>
05:42:33.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.294839 31384 fs_manager.cc:359] Metadata directory not provided
05:42:33.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.295142 31384 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:42:33.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.306722 31384 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.006s sys 0.000s
05:42:33.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.312960 31402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:42:33.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.314654 31384 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
05:42:33.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.315078 31384 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data,/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/wal
05:42:33.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17be7a6354c946de8fc6484884d0dfef"
05:42:33.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:28 on dist-test-slave-fjm6"
05:42:33.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.319128 31384 fs_report.cc:383] FS layout report
05:42:33.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:42:33.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/wal
05:42:33.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/wal
05:42:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data/data
05:42:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:42:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:42:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:42:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:42:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:42:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:42:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:42:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:42:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:42:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:42:33.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:42:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.367388 31384 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:42:33.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.375514 31384 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:42:33.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.376171 31384 kserver.cc:163] Server-wide thread pool size limit: 3276
05:42:33.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.474205 31384 rpc_server.cc:307] RPC server started. Bound to: 127.30.73.252:36847
05:42:33.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.474326 31453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.73.252:36847 every 8 connection(s)
05:42:33.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.477240 31384 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-2/data/info.pb
05:42:33.479 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:42:33.480 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.73.253:35861
05:42:33.480 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
05:42:33.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.478305 31015 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu as pid 31384
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.484421 31015 external_mini_cluster.cc:1326] Running /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/logs
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data/info.pb
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:42:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.73.253:35861
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.73.253
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33235
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.253.keytab
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.73.253
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.73.212:35393
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.73.254:33133,127.30.73.253:35861,127.30.73.252:36847
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:33.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36121/jwks.json
05:42:33.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:42:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.494987 31454 sys_catalog.cc:263] Verifying existing consensus state
05:42:33.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.503193 31454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: Bootstrap starting.
05:42:33.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.551980 31454 log.cc:824] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: Log is configured to *not* fsync() on all Append() calls
05:42:33.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.557304 31454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
05:42:33.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.558385 31454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: Bootstrap complete.
05:42:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.588250 31454 raft_consensus.cc:348] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } } peers { permanent_uuid: "62f2f856c0b749c8bea4bf839de07509" member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } } peers { permanent_uuid: "17be7a6354c946de8fc6484884d0dfef" member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } }
05:42:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.590983 31454 raft_consensus.cc:374] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:42:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.591359 31454 raft_consensus.cc:729] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 17be7a6354c946de8fc6484884d0dfef, State: Initialized, Role: FOLLOWER
05:42:33.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.592347 31454 consensus_queue.cc:260] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } } peers { permanent_uuid: "62f2f856c0b749c8bea4bf839de07509" member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } } peers { permanent_uuid: "17be7a6354c946de8fc6484884d0dfef" member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } }
05:42:33.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.600464 31460 sys_catalog.cc:455] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } } peers { permanent_uuid: "62f2f856c0b749c8bea4bf839de07509" member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } } peers { permanent_uuid: "17be7a6354c946de8fc6484884d0dfef" member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } } }
05:42:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.601724 31460 sys_catalog.cc:458] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [sys.catalog]: This master's current role is: FOLLOWER
05:42:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.602396 31454 sys_catalog.cc:564] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [sys.catalog]: configured and running, proceeding with master startup.
05:42:33.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:33.632927 31408 tablet.cc:2369] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:42:33.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:33.647728 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:33.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:33.648092 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:33.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.655939 31384 master_runner.cc:186] Error getting master registration for 127.30.73.254:33133: Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111),
05:42:33.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.688725 31384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:42:33.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.960162 31460 raft_consensus.cc:482] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:42:33.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.960786 31460 raft_consensus.cc:504] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } } peers { permanent_uuid: "62f2f856c0b749c8bea4bf839de07509" member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } } peers { permanent_uuid: "17be7a6354c946de8fc6484884d0dfef" member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } }
05:42:33.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:33.965863 31405 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111)
05:42:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.970530 31460 leader_election.cc:288] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 293f1cfbf9d24044ad944ef86b1f760c (127.30.73.254:33133), 62f2f856c0b749c8bea4bf839de07509 (127.30.73.253:35861)
05:42:33.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:33.978667 31457 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:42:33.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:33.979650 31457 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:42:33.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:33.980872 31457 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:42:33.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:33.984026 31404 leader_election.cc:334] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 62f2f856c0b749c8bea4bf839de07509 (127.30.73.253:35861): Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:33.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:33.986395 31405 leader_election.cc:334] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 293f1cfbf9d24044ad944ef86b1f760c (127.30.73.254:33133): Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111)
05:42:33.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.986807 31405 leader_election.cc:302] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 17be7a6354c946de8fc6484884d0dfef; no voters: 293f1cfbf9d24044ad944ef86b1f760c, 62f2f856c0b749c8bea4bf839de07509
05:42:33.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:33.989882 31460 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:42:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:34.036465 31457 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:42:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:34.036795 31457 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:42:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:34.037248 31457 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:42:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:34.037581 31457 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:42:34.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:34.037904 31457 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:42:34.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:34.038239 31457 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:42:34.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:34.038496 31457 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36121/jwks.json
05:42:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:34.101646 31457 master_runner.cc:386] Master server non-default flags:
05:42:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:42:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.73.212:35393
05:42:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:42:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:42:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data
05:42:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal
05:42:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:42:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:42:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.73.254:33133,127.30.73.253:35861,127.30.73.252:36847
05:42:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/krb5kdc/kudu_127.30.73.253.keytab
05:42:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.73.253
05:42:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.73.253:35861
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36121/jwks.json
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data/info.pb
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.73.253
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33235
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/logs
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:42:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:42:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:42:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:42:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:42:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:42:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:42:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 04:41:40 UTC on ccycloud.quasar-bblvbx.root.comops.site
05:42:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671828
05:42:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:42:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:34.103219 31457 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:42:34.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:34.104600 31457 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:42:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:34.158435 31480 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:34.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:34.664070 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:34.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:34.664710 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:35.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:35.284849 30859 debug-util.cc:398] Leaking SignalData structure 0x7b08000afb80 after lost signal to thread 30735
05:42:35.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:35.285943 30859 debug-util.cc:398] Leaking SignalData structure 0x7b08000be180 after lost signal to thread 30862
05:42:35.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:35.674932 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:35.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:35.675592 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:34.162045 31484 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:35.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:34.167948 31481 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:42:35.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:34.160207 31457 server_base.cc:1034] running on GCE node
05:42:35.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.687216 31457 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:42:35.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.691239 31457 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:42:35.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.692782 31457 hybrid_clock.cc:648] HybridClock initialized: now 1790314955692776 us; error 97 us; skew 500 ppm
05:42:35.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.694060 31457 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:42:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:35 dist-test-slave-fjm6 krb5kdc[31027](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314955, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:42:35.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.703589 31457 init.cc:377] Logged in from keytab as kudu/127.30.73.253@KRBTEST.COM (short username kudu)
05:42:35.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.709793 31457 webserver.cc:460] Webserver started at http://127.30.73.253:33235/ using document root <none> and password file <none>
05:42:35.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.711016 31457 fs_manager.cc:359] Metadata directory not provided
05:42:35.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.711333 31457 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:42:35.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.722782 31457 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.005s sys 0.000s
05:42:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.744025 31492 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:42:35.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.747228 31457 fs_manager.cc:725] Time spent opening block manager: real 0.021s user 0.002s sys 0.016s
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.747701 31457 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data,/tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62f2f856c0b749c8bea4bf839de07509"
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 05:42:25 on dist-test-slave-fjm6"
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.751240 31457 fs_report.cc:383] FS layout report
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data/data
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:42:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:42:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:42:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:42:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:42:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:42:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.765111 31493 raft_consensus.cc:482] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:42:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.766145 31493 raft_consensus.cc:504] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } } peers { permanent_uuid: "62f2f856c0b749c8bea4bf839de07509" member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } } peers { permanent_uuid: "17be7a6354c946de8fc6484884d0dfef" member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } }
05:42:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.782068 31493 leader_election.cc:288] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 293f1cfbf9d24044ad944ef86b1f760c (127.30.73.254:33133), 62f2f856c0b749c8bea4bf839de07509 (127.30.73.253:35861)
05:42:35.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:35.802776 31404 leader_election.cc:334] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 62f2f856c0b749c8bea4bf839de07509 (127.30.73.253:35861): Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:35.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:35.806998 31405 leader_election.cc:334] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 293f1cfbf9d24044ad944ef86b1f760c (127.30.73.254:33133): Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111)
05:42:35.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.807837 31405 leader_election.cc:302] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 17be7a6354c946de8fc6484884d0dfef; no voters: 293f1cfbf9d24044ad944ef86b1f760c, 62f2f856c0b749c8bea4bf839de07509
05:42:35.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.808826 31493 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:42:35.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.837656 31457 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:42:35.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.846006 31457 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:42:35.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:35.846653 31457 kserver.cc:163] Server-wide thread pool size limit: 3276
05:42:36.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:36.009091 31457 rpc_server.cc:307] RPC server started. Bound to: 127.30.73.253:35861
05:42:36.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:36.009209 31549 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.73.253:35861 every 8 connection(s)
05:42:36.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:36.014487 31457 server_base.cc:1166] Dumped server information to /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/data/info.pb
05:42:36.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:36.023452 31015 external_mini_cluster.cc:1388] Started /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu as pid 31457
05:42:36.024 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:42:36.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:36.037315 31550 sys_catalog.cc:263] Verifying existing consensus state
05:42:36.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260925 05:42:36.038271 31550 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:42:36.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:36.038837 31457 master.cc:537] Master@127.30.73.253:35861 shutting down...
05:42:36.133 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:36.149075 31457 master.cc:552] Master@127.30.73.253:35861 shutdown complete.
05:42:36.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_t1FW1/test-tmp/mini-kudu-cluster4113371503733547077/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:42:36.264 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.73.252:36847(127.30.73.252:36847)] connecting to peer
05:42:36.266 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:36.364 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.30.73.252:36847(127.30.73.252:36847)] Successfully connected to peer
05:42:36.370 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.372 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.373 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.376 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.377 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.379 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.486 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
05:42:36.502 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.30.73.252
05:42:36.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:36.686800 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:36.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:36.687493 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:36.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:36 dist-test-slave-fjm6 krb5kdc[31027](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314949, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.73.252@KRBTEST.COM
05:42:36.802 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xd5976a54, L:/127.0.0.1:34408 - R:/127.30.73.252:36847] using SASL/GSSAPI
05:42:36.823 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:36.841 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:36.845 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.846 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:36.846 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.848 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.848 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.852 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:36.860 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:36.864 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:36.867 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.868 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.868 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.870 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.870 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:36.879 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:36.881 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:36.883 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.883 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.884 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.885 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.886 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.888 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:36.900 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:36.902 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:36.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.906 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.908 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:36.919 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:36.921 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:36.926 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.927 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.927 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.927 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.928 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.928 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:36.960 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:36.961 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:36.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:36.967 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.967 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:36.970 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:37.059 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:37.061 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:37.065 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:37.066 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:37.066 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:37.066 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:37.067 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:37.067 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:37.069 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:37.300 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:37.302 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:37.302 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:37.302 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:37.304 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:37.304 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:37.304 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:37.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:37.309 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:37.460 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:37.462 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:37.464 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:37.464 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:37.465 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:37.465 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:37.466 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:37.466 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:37.468 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:37.580 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:37.581 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:37.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:37.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:37.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:37.586 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:37.586 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:37.586 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:37.587 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:37.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:37.690424 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:37.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:37.690687 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:38.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:38.255244 31558 raft_consensus.cc:482] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:42:38.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:38.255671 31558 raft_consensus.cc:504] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } } peers { permanent_uuid: "62f2f856c0b749c8bea4bf839de07509" member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } } peers { permanent_uuid: "17be7a6354c946de8fc6484884d0dfef" member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } }
05:42:38.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:38.257223 31558 leader_election.cc:288] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 293f1cfbf9d24044ad944ef86b1f760c (127.30.73.254:33133), 62f2f856c0b749c8bea4bf839de07509 (127.30.73.253:35861)
05:42:38.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:38.262509 31405 leader_election.cc:334] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 293f1cfbf9d24044ad944ef86b1f760c (127.30.73.254:33133): Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111)
05:42:38.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:38.262923 31404 leader_election.cc:334] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 62f2f856c0b749c8bea4bf839de07509 (127.30.73.253:35861): Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:38.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:38.263247 31404 leader_election.cc:302] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 17be7a6354c946de8fc6484884d0dfef; no voters: 293f1cfbf9d24044ad944ef86b1f760c, 62f2f856c0b749c8bea4bf839de07509
05:42:38.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:38.263932 31558 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:42:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:38.692252 31384 master_runner.cc:186] Error getting master registration for 127.30.73.254:33133: Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111),
05:42:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:38.692663 31384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:42:38.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:38.694564 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:38.694825 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:39.560 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:39.562 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:39.563 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:39.563 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:39.564 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:39.564 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:39.564 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:39.564 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:39.566 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:39.697772 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:39.698114 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:40.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:40.701377 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:40.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:40.701619 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:40.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:40.922628 30719 debug-util.cc:398] Leaking SignalData structure 0x7b08000afd60 after lost signal to thread 30589
05:42:40.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:40.923450 30719 debug-util.cc:398] Leaking SignalData structure 0x7b08000af3c0 after lost signal to thread 30722
05:42:41.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:41.705276 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:41.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:41.705519 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:42.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:42.182710 31565 raft_consensus.cc:482] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:42:42.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:42.183458 31565 raft_consensus.cc:504] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } } peers { permanent_uuid: "62f2f856c0b749c8bea4bf839de07509" member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } } peers { permanent_uuid: "17be7a6354c946de8fc6484884d0dfef" member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } }
05:42:42.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:42.185583 31565 leader_election.cc:288] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 293f1cfbf9d24044ad944ef86b1f760c (127.30.73.254:33133), 62f2f856c0b749c8bea4bf839de07509 (127.30.73.253:35861)
05:42:42.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:42.203600 31404 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111) [suppressed 5 similar messages]
05:42:42.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:42.207912 31404 leader_election.cc:334] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 62f2f856c0b749c8bea4bf839de07509 (127.30.73.253:35861): Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:42.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:42.208624 31405 leader_election.cc:334] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 293f1cfbf9d24044ad944ef86b1f760c (127.30.73.254:33133): Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111)
05:42:42.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:42.208943 31405 leader_election.cc:302] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 17be7a6354c946de8fc6484884d0dfef; no voters: 293f1cfbf9d24044ad944ef86b1f760c, 62f2f856c0b749c8bea4bf839de07509
05:42:42.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:42.209676 31565 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:42:42.680 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:42.682 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:42.683 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:42.684 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:42.684 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:42.684 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:42.685 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:42.685 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:42.686 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:42.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:42.710145 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:42.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:42.710470 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:43.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:43 dist-test-slave-fjm6 krb5kdc[31027](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314963, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:42:43.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:43.291105 31398 init.cc:295] Successfully reacquired a new kerberos TGT
05:42:43.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:43.695420 31384 master_runner.cc:186] Error getting master registration for 127.30.73.254:33133: Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111),
05:42:43.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:43.695883 31384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:42:43.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:43.713189 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:43.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:43.713444 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:44.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:44.716231 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:44.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:44.716550 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:45.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:45.719627 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:45.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:45.719933 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:46.019 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:46.021 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:46.023 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:46.023 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:46.023 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:46.024 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:46.024 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:46.024 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:46.025 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:46.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:46.722622 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:46.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:46.722868 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:47.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:47.146673 31000 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 30875
05:42:47.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:47.147598 31000 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb040 after lost signal to thread 31003
05:42:47.340 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:47.342 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:47.343 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:47.344 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:47.344 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:47.346 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:47.346 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:47.346 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:47.347 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:47.727213 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:47.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:47.727993 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:48.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:48.533726 30719 debug-util.cc:398] Leaking SignalData structure 0x7b08000af140 after lost signal to thread 30589
05:42:48.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:48.534850 30719 debug-util.cc:398] Leaking SignalData structure 0x7b08000af220 after lost signal to thread 30722
05:42:48.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:48.698598 31384 master_runner.cc:186] Error getting master registration for 127.30.73.254:33133: Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111),
05:42:48.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:48.699155 31384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:42:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:48.731745 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:48.732043 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:48.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:48.933619 31577 raft_consensus.cc:482] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:42:48.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:48.934022 31577 raft_consensus.cc:504] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } } peers { permanent_uuid: "62f2f856c0b749c8bea4bf839de07509" member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } } peers { permanent_uuid: "17be7a6354c946de8fc6484884d0dfef" member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } }
05:42:48.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:48.935807 31577 leader_election.cc:288] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 293f1cfbf9d24044ad944ef86b1f760c (127.30.73.254:33133), 62f2f856c0b749c8bea4bf839de07509 (127.30.73.253:35861)
05:42:48.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:48.937161 31405 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111) [suppressed 1 similar messages]
05:42:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:48.940356 31405 leader_election.cc:334] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 293f1cfbf9d24044ad944ef86b1f760c (127.30.73.254:33133): Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111)
05:42:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:48.940791 31404 leader_election.cc:334] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 62f2f856c0b749c8bea4bf839de07509 (127.30.73.253:35861): Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:48.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:48.941200 31404 leader_election.cc:302] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 17be7a6354c946de8fc6484884d0dfef; no voters: 293f1cfbf9d24044ad944ef86b1f760c, 62f2f856c0b749c8bea4bf839de07509
05:42:48.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:48.941943 31577 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:42:49.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:49.735277 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:49.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:49.735561 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:50.560 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:50.561 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:50.563 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:50.563 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:50.564 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:50.564 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:50.564 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:50.564 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:50.565 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:50.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:50.738400 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:50.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:50.738652 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:51.539 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:51.541 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:51.543 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:51.543 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:51.543 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:51.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:51.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:51.545 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:51.546 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:51.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:51.741751 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:51.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:51.741990 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:52.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:52.745954 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:52.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:52.746390 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:53.200 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:53.201 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:53.203 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:53.203 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:53.203 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:53.203 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:53.204 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:53.204 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:53.206 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:53.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 05:42:53 dist-test-slave-fjm6 krb5kdc[31027](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790314973, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.73.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:42:53.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:53.293609 31398 init.cc:295] Successfully reacquired a new kerberos TGT
05:42:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:53.702849 31384 master_runner.cc:186] Error getting master registration for 127.30.73.254:33133: Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111),
05:42:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:53.703408 31384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:42:53.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:53.749091 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:53.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:53.749337 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:54.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:54.752245 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:54.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:54.752583 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:55.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:55.397265 31586 raft_consensus.cc:482] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:42:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:55.397691 31586 raft_consensus.cc:504] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "293f1cfbf9d24044ad944ef86b1f760c" member_type: VOTER last_known_addr { host: "127.30.73.254" port: 33133 } } peers { permanent_uuid: "62f2f856c0b749c8bea4bf839de07509" member_type: VOTER last_known_addr { host: "127.30.73.253" port: 35861 } } peers { permanent_uuid: "17be7a6354c946de8fc6484884d0dfef" member_type: VOTER last_known_addr { host: "127.30.73.252" port: 36847 } }
05:42:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:55.399358 31586 leader_election.cc:288] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 293f1cfbf9d24044ad944ef86b1f760c (127.30.73.254:33133), 62f2f856c0b749c8bea4bf839de07509 (127.30.73.253:35861)
05:42:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:55.401388 31405 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111) [suppressed 1 similar messages]
05:42:55.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:55.404366 31404 leader_election.cc:334] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 62f2f856c0b749c8bea4bf839de07509 (127.30.73.253:35861): Network error: Client connection negotiation failed: client connection to 127.30.73.253:35861: connect: Connection refused (error 111)
05:42:55.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:55.404815 31405 leader_election.cc:334] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 293f1cfbf9d24044ad944ef86b1f760c (127.30.73.254:33133): Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111)
05:42:55.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:55.405095 31405 leader_election.cc:302] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 17be7a6354c946de8fc6484884d0dfef; no voters: 293f1cfbf9d24044ad944ef86b1f760c, 62f2f856c0b749c8bea4bf839de07509
05:42:55.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:55.405607 31586 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:42:55.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:55.755545 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:55.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:55.755834 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:56.759071 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:56.759387 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:57.260 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:57.262 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:57.263 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:57.263 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:57.264 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:57.264 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:57.264 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:57.265 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:57.266 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:57.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:57.762346 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:57.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:57.762637 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:58.705929 31384 master_runner.cc:186] Error getting master registration for 127.30.73.254:33133: Network error: Client connection negotiation failed: client connection to 127.30.73.254:33133: connect: Connection refused (error 111),
05:42:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:42:58.706269 31384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:42:58.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:58.766134 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:58.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:58.766441 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:59.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:59.770057 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:42:59.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:42:59.770339 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:42:59.800 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:59.801 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:59.802 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:59.803 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:59.803 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:59.803 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:59.803 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:59.803 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:59.805 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:42:59.940 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:42:59.941 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:42:59.942 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:59.942 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:59.942 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:42:59.943 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:59.943 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:59.943 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:42:59.944 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:43:00.200 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:43:00.201 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:43:00.202 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:43:00.202 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:43:00.202 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:43:00.203 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:43:00.203 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:43:00.203 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:43:00.205 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:43:00.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:43:00.644976 30719 debug-util.cc:398] Leaking SignalData structure 0x7b08000b46c0 after lost signal to thread 30589
05:43:00.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:43:00.645994 30719 debug-util.cc:398] Leaking SignalData structure 0x7b08000afc20 after lost signal to thread 30722
05:43:00.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:43:00.773705 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:43:00.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:43:00.774055 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:43:01.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:43:01.777382 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:43:01.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:43:01.777678 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:43:02.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:43:02.783547 31471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 17be7a6354c946de8fc6484884d0dfef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:43:02.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 05:43:02.784220 31471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:43:02.859 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.254:33133(127.30.73.254:33133)] connecting to peer
05:43:02.860 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.73.253:35861(127.30.73.253:35861)] connecting to peer
05:43:02.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:43:02.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.73.253:35861(127.30.73.253:35861)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:43:02.861 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:43:02.861 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.253:35861: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861
05:43:02.861 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.254:33133(127.30.73.254:33133)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:43:02.862 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.73.254:33133: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
05:43:02.864 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861; will retry
05:43:02.869 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26835), Traces: [0ms] refreshing cache from master, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [328ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [332ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [774ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [779ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [800ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [803ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [806ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [814ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [823ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [823ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [824ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [826ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [834ms] refreshing cache from master, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [838ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [840ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [842ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [843ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [860ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [861ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [863ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [864ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [874ms] refreshing cache from master, [874ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [881ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [881ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [882ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [883ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [914ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [920ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [923ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [925ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [926ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1014ms] refreshing cache from master, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1254ms] refreshing cache from master, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1257ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1263ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1264ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1414ms] refreshing cache from master, [1415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1419ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1423ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1535ms] refreshing cache from master, [1535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1540ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1541ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1542ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1542ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [3514ms] refreshing cache from master, [3515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [3516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [3516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [3518ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [3519ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [3520ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [3521ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [6635ms] refreshing cache from master, [6635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [6636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [6636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [6639ms] too many traces: truncated at 100 traces, deferred=Deferred@1897804752(state=PENDING, result=null, callback=(continuation of Deferred@696425352 after retry RPC after error@42731196) -> (continuation of Deferred@1890845386 after retry RPC after error@1890856895) -> (continuation of Deferred@1305978106 after retry RPC after error@789377562) -> (continuation of Deferred@2147354467 after retry RPC after error@786831092) -> (continuation of Deferred@1592166310 after retry RPC after error@372388418) -> (continuation of Deferred@427354872 after retry RPC after error@129441429) -> (continuation of Deferred@1829139058 after retry RPC after error@1780612530) -> (continuation of Deferred@877716772 after retry RPC after error@195823245) -> (continuation of Deferred@1432842344 after retry RPC after error@810474267) -> (continuation of Deferred@50622963 after retry RPC after error@1761056624) -> (continuation of Deferred@108879414 after retry RPC after error@1076747233) -> (continuation of Deferred@1611051518 after retry RPC after error@874444441) -> (continuation of Deferred@1493074626 after retry RPC after error@1981987055) -> (continuation of Deferred@595488404 after retry RPC after error@1620573788) -> (continuation of Deferred@1280542750 after retry RPC after error@794141883) -> (continuation of Deferred@2140108043 after retry RPC after error@540344760) -> (continuation of Deferred@1895202520 after retry RPC after error@1921285121) -> (continuation of Deferred@1990686839 after retry RPC after error@2123861978) -> (continuation of Deferred@1028931182 after retry RPC after error@1818311089) -> (continuation of Deferred@558399626 after retry RPC after error@1667441190) -> (continuation of Deferred@2144913258 after retry RPC after error@445765009) -> (continuation of Deferred@2134399641 after retry RPC after error@87045450), errback=(continuation of Deferred@696425352 after retry RPC after error@42731196) -> (continuation of Deferred@1890845386 after retry RPC after error@1890856895) -> (continuation of Deferred@1305978106 after retry RPC after error@789377562) -> (continuation of Deferred@2147354467 after retry RPC after error@786831092) -> (continuation of Deferred@1592166310 after retry RPC after error@372388418) -> (continuation of Deferred@427354872 after retry RPC after error@129441429) -> (continuation of Deferred@1829139058 after retry RPC after error@1780612530) -> (continuation of Deferred@877716772 after retry RPC after error@195823245) -> (continuation of Deferred@1432842344 after retry RPC after error@810474267) -> (continuation of Deferred@50622963 after retry RPC after error@1761056624) -> (continuation of Deferred@108879414 after retry RPC after error@1076747233) -> (continuation of Deferred@1611051518 after retry RPC after error@874444441) -> (continuation of Deferred@1493074626 after retry RPC after error@1981987055) -> (continuation of Deferred@595488404 after retry RPC after error@1620573788) -> (continuation of Deferred@1280542750 after retry RPC after error@794141883) -> (continuation of Deferred@2140108043 after retry RPC after error@540344760) -> (continuation of Deferred@1895202520 after retry RPC after error@1921285121) -> (continuation of Deferred@1990686839 after retry RPC after error@2123861978) -> (continuation of Deferred@1028931182 after retry RPC after error@1818311089) -> (continuation of Deferred@558399626 after retry RPC after error@1667441190) -> (continuation of Deferred@2144913258 after retry RPC after error@445765009) -> (continuation of Deferred@2134399641 after retry RPC after error@87045450)))
05:43:02.873 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.30.73.252:36847(127.30.73.252:36847)] disconnecting while in state READY
05:43:02.874 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.30.73.254:33133(127.30.73.254:33133)] disconnecting while in state TERMINATED
05:43:02.875 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.30.73.253:35861(127.30.73.253:35861)] disconnecting while in state TERMINATED
05:43:02.881 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:43:02.884 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:43:02.884 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.30.73.252:36847(127.30.73.252:36847)] handling channelInactive
05:43:02.885 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.73.252:36847(127.30.73.252:36847)] cleaning up while in state READY due to: connection closed
05:43:02.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:43:02.893373 31015 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu with pid 31384
05:43:04.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T05:43:04Z chronyd exiting
05:43:05.108 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:43:05.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:43:05.110004 30288 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu with pid 30569
05:43:05.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:43:05.140053 30288 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu with pid 30727
05:43:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:43:05.170498 30288 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu with pid 30873
05:43:05.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:43:05.201264 30288 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu with pid 30300
05:43:05.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:43:05.233884 30288 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu with pid 30370
05:43:05.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 05:43:05.268069 30288 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task_t1FW1/build/tsan/bin/kudu with pid 30445
05:43:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T05:43:06Z chronyd exiting
05:43:07.498 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26835), Traces: [0ms] refreshing cache from master, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [328ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [332ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [774ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [779ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [800ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [803ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [806ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [814ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [823ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [823ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [824ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [826ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [834ms] refreshing cache from master, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [838ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [840ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [842ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [843ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [860ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [861ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [863ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [864ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [874ms] refreshing cache from master, [874ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [881ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [881ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [882ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [883ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [914ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [920ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [923ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [925ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [926ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1014ms] refreshing cache from master, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1254ms] refreshing cache from master, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1257ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1263ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1264ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1414ms] refreshing cache from master, [1415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1419ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1423ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1535ms] refreshing cache from master, [1535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1540ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1541ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1542ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1542ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [3514ms] refreshing cache from master, [3515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [3516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [3516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [3518ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [3519ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [3520ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [3521ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [6635ms] refreshing cache from master, [6635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [6636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [6636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [6639ms] too many traces: truncated at 100 traces, deferred=Deferred@1897804752(state=PENDING, result=null, callback=(continuation of Deferred@696425352 after retry RPC after error@42731196) -> (continuation of Deferred@1890845386 after retry RPC after error@1890856895) -> (continuation of Deferred@1305978106 after retry RPC after error@789377562) -> (continuation of Deferred@2147354467 after retry RPC after error@786831092) -> (continuation of Deferred@1592166310 after retry RPC after error@372388418) -> (continuation of Deferred@427354872 after retry RPC after error@129441429) -> (continuation of Deferred@1829139058 after retry RPC after error@1780612530) -> (continuation of Deferred@877716772 after retry RPC after error@195823245) -> (continuation of Deferred@1432842344 after retry RPC after error@810474267) -> (continuation of Deferred@50622963 after retry RPC after error@1761056624) -> (continuation of Deferred@108879414 after retry RPC after error@1076747233) -> (continuation of Deferred@1611051518 after retry RPC after error@874444441) -> (continuation of Deferred@1493074626 after retry RPC after error@1981987055) -> (continuation of Deferred@595488404 after retry RPC after error@1620573788) -> (continuation of Deferred@1280542750 after retry RPC after error@794141883) -> (continuation of Deferred@2140108043 after retry RPC after error@540344760) -> (continuation of Deferred@1895202520 after retry RPC after error@1921285121) -> (continuation of Deferred@1990686839 after retry RPC after error@2123861978) -> (continuation of Deferred@1028931182 after retry RPC after error@1818311089) -> (continuation of Deferred@558399626 after retry RPC after error@1667441190) -> (continuation of Deferred@2144913258 after retry RPC after error@445765009) -> (continuation of Deferred@2134399641 after retry RPC after error@87045450), errback=(continuation of Deferred@696425352 after retry RPC after error@42731196) -> (continuation of Deferred@1890845386 after retry RPC after error@1890856895) -> (continuation of Deferred@1305978106 after retry RPC after error@789377562) -> (continuation of Deferred@2147354467 after retry RPC after error@786831092) -> (continuation of Deferred@1592166310 after retry RPC after error@372388418) -> (continuation of Deferred@427354872 after retry RPC after error@129441429) -> (continuation of Deferred@1829139058 after retry RPC after error@1780612530) -> (continuation of Deferred@877716772 after retry RPC after error@195823245) -> (continuation of Deferred@1432842344 after retry RPC after error@810474267) -> (continuation of Deferred@50622963 after retry RPC after error@1761056624) -> (continuation of Deferred@108879414 after retry RPC after error@1076747233) -> (continuation of Deferred@1611051518 after retry RPC after error@874444441) -> (continuation of Deferred@1493074626 after retry RPC after error@1981987055) -> (continuation of Deferred@595488404 after retry RPC after error@1620573788) -> (continuation of Deferred@1280542750 after retry RPC after error@794141883) -> (continuation of Deferred@2140108043 after retry RPC after error@540344760) -> (continuation of Deferred@1895202520 after retry RPC after error@1921285121) -> (continuation of Deferred@1990686839 after retry RPC after error@2123861978) -> (continuation of Deferred@1028931182 after retry RPC after error@1818311089) -> (continuation of Deferred@558399626 after retry RPC after error@1667441190) -> (continuation of Deferred@2144913258 after retry RPC after error@445765009) -> (continuation of Deferred@2134399641 after retry RPC after error@87045450)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.30.73.254:33133
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26835), Traces: [0ms] refreshing cache from master, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [328ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [332ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [774ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [779ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [800ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [803ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [806ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [814ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [823ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [823ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [824ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [826ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [834ms] refreshing cache from master, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [838ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [840ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [842ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [843ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [860ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [861ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [863ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [864ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [874ms] refreshing cache from master, [874ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [881ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [881ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [882ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [883ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [914ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [920ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [923ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [925ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [926ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1014ms] refreshing cache from master, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1254ms] refreshing cache from master, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1257ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1263ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1264ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1414ms] refreshing cache from master, [1415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1419ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1423ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1535ms] refreshing cache from master, [1535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1540ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1541ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1542ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1542ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [3514ms] refreshing cache from master, [3515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [3516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [3516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [3518ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [3519ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [3520ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [3521ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [6635ms] refreshing cache from master, [6635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [6636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [6636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [6639ms] too many traces: truncated at 100 traces, deferred=Deferred@1897804752(state=PENDING, result=null, callback=(continuation of Deferred@696425352 after retry RPC after error@42731196) -> (continuation of Deferred@1890845386 after retry RPC after error@1890856895) -> (continuation of Deferred@1305978106 after retry RPC after error@789377562) -> (continuation of Deferred@2147354467 after retry RPC after error@786831092) -> (continuation of Deferred@1592166310 after retry RPC after error@372388418) -> (continuation of Deferred@427354872 after retry RPC after error@129441429) -> (continuation of Deferred@1829139058 after retry RPC after error@1780612530) -> (continuation of Deferred@877716772 after retry RPC after error@195823245) -> (continuation of Deferred@1432842344 after retry RPC after error@810474267) -> (continuation of Deferred@50622963 after retry RPC after error@1761056624) -> (continuation of Deferred@108879414 after retry RPC after error@1076747233) -> (continuation of Deferred@1611051518 after retry RPC after error@874444441) -> (continuation of Deferred@1493074626 after retry RPC after error@1981987055) -> (continuation of Deferred@595488404 after retry RPC after error@1620573788) -> (continuation of Deferred@1280542750 after retry RPC after error@794141883) -> (continuation of Deferred@2140108043 after retry RPC after error@540344760) -> (continuation of Deferred@1895202520 after retry RPC after error@1921285121) -> (continuation of Deferred@1990686839 after retry RPC after error@2123861978) -> (continuation of Deferred@1028931182 after retry RPC after error@1818311089) -> (continuation of Deferred@558399626 after retry RPC after error@1667441190) -> (continuation of Deferred@2144913258 after retry RPC after error@445765009) -> (continuation of Deferred@2134399641 after retry RPC after error@87045450), errback=(continuation of Deferred@696425352 after retry RPC after error@42731196) -> (continuation of Deferred@1890845386 after retry RPC after error@1890856895) -> (continuation of Deferred@1305978106 after retry RPC after error@789377562) -> (continuation of Deferred@2147354467 after retry RPC after error@786831092) -> (continuation of Deferred@1592166310 after retry RPC after error@372388418) -> (continuation of Deferred@427354872 after retry RPC after error@129441429) -> (continuation of Deferred@1829139058 after retry RPC after error@1780612530) -> (continuation of Deferred@877716772 after retry RPC after error@195823245) -> (continuation of Deferred@1432842344 after retry RPC after error@810474267) -> (continuation of Deferred@50622963 after retry RPC after error@1761056624) -> (continuation of Deferred@108879414 after retry RPC after error@1076747233) -> (continuation of Deferred@1611051518 after retry RPC after error@874444441) -> (continuation of Deferred@1493074626 after retry RPC after error@1981987055) -> (continuation of Deferred@595488404 after retry RPC after error@1620573788) -> (continuation of Deferred@1280542750 after retry RPC after error@794141883) -> (continuation of Deferred@2140108043 after retry RPC after error@540344760) -> (continuation of Deferred@1895202520 after retry RPC after error@1921285121) -> (continuation of Deferred@1990686839 after retry RPC after error@2123861978) -> (continuation of Deferred@1028931182 after retry RPC after error@1818311089) -> (continuation of Deferred@558399626 after retry RPC after error@1667441190) -> (continuation of Deferred@2144913258 after retry RPC after error@445765009) -> (continuation of Deferred@2134399641 after retry RPC after error@87045450)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.30.73.254:33133
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
05:43:07.952 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
05:43:07.953 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26835), Traces: [0ms] refreshing cache from master, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [328ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [332ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [774ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [779ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [800ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [803ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [806ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [814ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [823ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [823ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [824ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [826ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [834ms] refreshing cache from master, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [838ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [840ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [842ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [843ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [860ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [861ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [863ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [864ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [874ms] refreshing cache from master, [874ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [881ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [881ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [882ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [883ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [914ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [920ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [923ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [925ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [926ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1014ms] refreshing cache from master, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1254ms] refreshing cache from master, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1257ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1263ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1264ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1414ms] refreshing cache from master, [1415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1419ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1423ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1535ms] refreshing cache from master, [1535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1540ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1541ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1542ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1542ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [3514ms] refreshing cache from master, [3515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [3516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [3516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [3518ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [3519ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [3520ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [3521ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [6635ms] refreshing cache from master, [6635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [6636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [6636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [6639ms] too many traces: truncated at 100 traces, deferred=Deferred@1897804752(state=PENDING, result=null, callback=(continuation of Deferred@696425352 after retry RPC after error@42731196) -> (continuation of Deferred@1890845386 after retry RPC after error@1890856895) -> (continuation of Deferred@1305978106 after retry RPC after error@789377562) -> (continuation of Deferred@2147354467 after retry RPC after error@786831092) -> (continuation of Deferred@1592166310 after retry RPC after error@372388418) -> (continuation of Deferred@427354872 after retry RPC after error@129441429) -> (continuation of Deferred@1829139058 after retry RPC after error@1780612530) -> (continuation of Deferred@877716772 after retry RPC after error@195823245) -> (continuation of Deferred@1432842344 after retry RPC after error@810474267) -> (continuation of Deferred@50622963 after retry RPC after error@1761056624) -> (continuation of Deferred@108879414 after retry RPC after error@1076747233) -> (continuation of Deferred@1611051518 after retry RPC after error@874444441) -> (continuation of Deferred@1493074626 after retry RPC after error@1981987055) -> (continuation of Deferred@595488404 after retry RPC after error@1620573788) -> (continuation of Deferred@1280542750 after retry RPC after error@794141883) -> (continuation of Deferred@2140108043 after retry RPC after error@540344760) -> (continuation of Deferred@1895202520 after retry RPC after error@1921285121) -> (continuation of Deferred@1990686839 after retry RPC after error@2123861978) -> (continuation of Deferred@1028931182 after retry RPC after error@1818311089) -> (continuation of Deferred@558399626 after retry RPC after error@1667441190) -> (continuation of Deferred@2144913258 after retry RPC after error@445765009) -> (continuation of Deferred@2134399641 after retry RPC after error@87045450), errback=(continuation of Deferred@696425352 after retry RPC after error@42731196) -> (continuation of Deferred@1890845386 after retry RPC after error@1890856895) -> (continuation of Deferred@1305978106 after retry RPC after error@789377562) -> (continuation of Deferred@2147354467 after retry RPC after error@786831092) -> (continuation of Deferred@1592166310 after retry RPC after error@372388418) -> (continuation of Deferred@427354872 after retry RPC after error@129441429) -> (continuation of Deferred@1829139058 after retry RPC after error@1780612530) -> (continuation of Deferred@877716772 after retry RPC after error@195823245) -> (continuation of Deferred@1432842344 after retry RPC after error@810474267) -> (continuation of Deferred@50622963 after retry RPC after error@1761056624) -> (continuation of Deferred@108879414 after retry RPC after error@1076747233) -> (continuation of Deferred@1611051518 after retry RPC after error@874444441) -> (continuation of Deferred@1493074626 after retry RPC after error@1981987055) -> (continuation of Deferred@595488404 after retry RPC after error@1620573788) -> (continuation of Deferred@1280542750 after retry RPC after error@794141883) -> (continuation of Deferred@2140108043 after retry RPC after error@540344760) -> (continuation of Deferred@1895202520 after retry RPC after error@1921285121) -> (continuation of Deferred@1990686839 after retry RPC after error@2123861978) -> (continuation of Deferred@1028931182 after retry RPC after error@1818311089) -> (continuation of Deferred@558399626 after retry RPC after error@1667441190) -> (continuation of Deferred@2144913258 after retry RPC after error@445765009) -> (continuation of Deferred@2134399641 after retry RPC after error@87045450)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.30.73.254:33133
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26835), Traces: [0ms] refreshing cache from master, [77ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [328ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [332ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [774ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [779ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [800ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [803ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [806ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [814ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [823ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [823ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [824ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [826ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [834ms] refreshing cache from master, [834ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [838ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [840ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [842ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [843ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [860ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [861ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [863ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [864ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [874ms] refreshing cache from master, [874ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [881ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [881ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [882ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [883ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [914ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [920ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [923ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [925ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [926ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1014ms] refreshing cache from master, [1014ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1021ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1254ms] refreshing cache from master, [1255ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1256ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1257ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1259ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1263ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1264ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1414ms] refreshing cache from master, [1415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1419ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1421ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1423ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1535ms] refreshing cache from master, [1535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [1536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [1536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [1540ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [1541ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [1542ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [1542ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [3514ms] refreshing cache from master, [3515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [3516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [3516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [3518ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.254:33133: Network error: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133, [3519ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.253:35861: Network error: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [3520ms] Sub RPC ConnectToMaster: received response from server master-127.30.73.252:36847: OK, [3521ms] delaying RPC due to: Service unavailable: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861, [6635ms] refreshing cache from master, [6635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.254:33133, [6636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.252:36847, [6636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.73.253:35861, [6639ms] too many traces: truncated at 100 traces, deferred=Deferred@1897804752(state=PENDING, result=null, callback=(continuation of Deferred@696425352 after retry RPC after error@42731196) -> (continuation of Deferred@1890845386 after retry RPC after error@1890856895) -> (continuation of Deferred@1305978106 after retry RPC after error@789377562) -> (continuation of Deferred@2147354467 after retry RPC after error@786831092) -> (continuation of Deferred@1592166310 after retry RPC after error@372388418) -> (continuation of Deferred@427354872 after retry RPC after error@129441429) -> (continuation of Deferred@1829139058 after retry RPC after error@1780612530) -> (continuation of Deferred@877716772 after retry RPC after error@195823245) -> (continuation of Deferred@1432842344 after retry RPC after error@810474267) -> (continuation of Deferred@50622963 after retry RPC after error@1761056624) -> (continuation of Deferred@108879414 after retry RPC after error@1076747233) -> (continuation of Deferred@1611051518 after retry RPC after error@874444441) -> (continuation of Deferred@1493074626 after retry RPC after error@1981987055) -> (continuation of Deferred@595488404 after retry RPC after error@1620573788) -> (continuation of Deferred@1280542750 after retry RPC after error@794141883) -> (continuation of Deferred@2140108043 after retry RPC after error@540344760) -> (continuation of Deferred@1895202520 after retry RPC after error@1921285121) -> (continuation of Deferred@1990686839 after retry RPC after error@2123861978) -> (continuation of Deferred@1028931182 after retry RPC after error@1818311089) -> (continuation of Deferred@558399626 after retry RPC after error@1667441190) -> (continuation of Deferred@2144913258 after retry RPC after error@445765009) -> (continuation of Deferred@2134399641 after retry RPC after error@87045450), errback=(continuation of Deferred@696425352 after retry RPC after error@42731196) -> (continuation of Deferred@1890845386 after retry RPC after error@1890856895) -> (continuation of Deferred@1305978106 after retry RPC after error@789377562) -> (continuation of Deferred@2147354467 after retry RPC after error@786831092) -> (continuation of Deferred@1592166310 after retry RPC after error@372388418) -> (continuation of Deferred@427354872 after retry RPC after error@129441429) -> (continuation of Deferred@1829139058 after retry RPC after error@1780612530) -> (continuation of Deferred@877716772 after retry RPC after error@195823245) -> (continuation of Deferred@1432842344 after retry RPC after error@810474267) -> (continuation of Deferred@50622963 after retry RPC after error@1761056624) -> (continuation of Deferred@108879414 after retry RPC after error@1076747233) -> (continuation of Deferred@1611051518 after retry RPC after error@874444441) -> (continuation of Deferred@1493074626 after retry RPC after error@1981987055) -> (continuation of Deferred@595488404 after retry RPC after error@1620573788) -> (continuation of Deferred@1280542750 after retry RPC after error@794141883) -> (continuation of Deferred@2140108043 after retry RPC after error@540344760) -> (continuation of Deferred@1895202520 after retry RPC after error@1921285121) -> (continuation of Deferred@1990686839 after retry RPC after error@2123861978) -> (continuation of Deferred@1028931182 after retry RPC after error@1818311089) -> (continuation of Deferred@558399626 after retry RPC after error@1667441190) -> (continuation of Deferred@2144913258 after retry RPC after error@445765009) -> (continuation of Deferred@2134399641 after retry RPC after error@87045450)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.30.73.254:33133,127.30.73.252:36847,127.30.73.253:35861) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.253:35861(127.30.73.253:35861): Connection refused: /127.30.73.253:35861,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.73.254:33133(127.30.73.254:33133): Connection refused: /127.30.73.254:33133
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.30.73.254:33133
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)