04:00:03.870 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
04:00:03.883 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskJlmI1R/test-tmp
04:00:03.885 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
04:00:03.889 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskJlmI1R/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
04:00:04.159 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
04:00:04.317 [DEBUG - main] (MiniKuduCluster.java:195) Response:
04:00:04.319 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
04:00:04.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:04.323076 1207 mini_oidc.cc:152] Starting JWKS server
04:00:04.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:04.327392 1207 webserver.cc:460] Webserver started at http://127.0.0.1:37033/ using document root <none> and password file <none>
04:00:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:04.327993 1207 mini_oidc.cc:189] Starting OIDC Discovery server
04:00:04.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:04.330142 1207 webserver.cc:460] Webserver started at http://127.0.0.1:32823/ using document root <none> and password file <none>
04:00:04.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T04:00:04Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
04:00:04.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T04:00:04Z Disabled control of system clock
04:00:04.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:04.387861 1207 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:04.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:04.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/wal
04:00:04.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/data
04:00:04.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:00:04.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:00:04.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:00:04.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:04.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:04.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:04.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/logs
04:00:04.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/data/info.pb
04:00:04.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:04.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:00:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:00:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:00:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:00:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:00:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:00:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.45.254:33177
04:00:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.45.254
04:00:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.45.212:46757
04:00:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.45.254:33177,127.1.45.253:38007,127.1.45.252:44305
04:00:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37033/jwks.json with env {}
04:00:04.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:04.698267 1219 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:00:04.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:04.698916 1219 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:00:04.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:04.699622 1219 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:00:04.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:04.730917 1219 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:00:04.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:04.731204 1219 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:00:04.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:04.731401 1219 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:00:04.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:04.731582 1219 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:00:04.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:04.731933 1219 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:00:04.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:04.732167 1219 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37033/jwks.json
04:00:04.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:04.767087 1219 master_runner.cc:386] Master server non-default flags:
04:00:04.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:04.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.45.212:46757
04:00:04.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:04.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:04.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/data
04:00:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/wal
04:00:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.45.254:33177,127.1.45.253:38007,127.1.45.252:44305
04:00:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.45.254:33177
04:00:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:00:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37033/jwks.json
04:00:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/data/info.pb
04:00:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.45.254
04:00:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:00:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:00:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:00:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/logs
04:00:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:00:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:00:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:00:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:00:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:00:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:00:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 03:03:38 UTC on ccycloud.quasar-murucv.root.comops.site
04:00:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454978
04:00:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:00:04.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:04.768448 1219 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:00:04.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:04.770582 1219 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:00:04.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:04.792410 1228 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:05.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:04.792441 1226 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:05.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:04.794294 1219 server_base.cc:1034] running on GCE node
04:00:05.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:04.792923 1225 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:06.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.002450 1219 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:00:06.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.005177 1219 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:00:06.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.006626 1219 hybrid_clock.cc:648] HybridClock initialized: now 1781841606006596 us; error 72 us; skew 500 ppm
04:00:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.007443 1219 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:00:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.017532 1219 webserver.cc:460] Webserver started at http://127.1.45.254:40941/ using document root <none> and password file <none>
04:00:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.018488 1219 fs_manager.cc:359] Metadata directory not provided
04:00:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.018735 1219 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:00:06.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.019209 1219 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:00:06.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.024255 1219 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/data/instance:
04:00:06.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8772b946b6e74e4abf5180ef7ccce19b"
04:00:06.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:06 on dist-test-slave-972x"
04:00:06.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.025484 1219 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/wal/instance:
04:00:06.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8772b946b6e74e4abf5180ef7ccce19b"
04:00:06.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:06 on dist-test-slave-972x"
04:00:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.033212 1219 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
04:00:06.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.039378 1235 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:00:06.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.040567 1219 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
04:00:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.040863 1219 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/data,/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/wal
04:00:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8772b946b6e74e4abf5180ef7ccce19b"
04:00:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:06 on dist-test-slave-972x"
04:00:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.041204 1219 fs_report.cc:383] FS layout report
04:00:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:00:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/wal
04:00:06.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/wal
04:00:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/data/data
04:00:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:00:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:00:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:00:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:00:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:00:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:00:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:00:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:00:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:00:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:00:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:00:06.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.085704 1219 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:00:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.087774 1219 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:00:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.088223 1219 kserver.cc:163] Server-wide thread pool size limit: 3276
04:00:06.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.168448 1219 rpc_server.cc:307] RPC server started. Bound to: 127.1.45.254:33177
04:00:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.168530 1286 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.45.254:33177 every 8 connection(s)
04:00:06.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.171470 1219 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/data/info.pb
04:00:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.175463 1207 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu as pid 1219
04:00:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.175997 1207 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-0/wal/instance
04:00:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.177630 1287 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.181543 1207 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/wal
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/data
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/logs
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/data/info.pb
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:00:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:00:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:00:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:00:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:00:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:00:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:06.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.45.253:38007
04:00:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.45.253
04:00:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.45.212:46757
04:00:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.45.254:33177,127.1.45.253:38007,127.1.45.252:44305
04:00:06.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:06.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37033/jwks.json with env {}
04:00:06.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.193240 1287 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } has no permanent_uuid. Determining permanent_uuid...
04:00:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.225073 1287 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } has no permanent_uuid. Determining permanent_uuid...
04:00:06.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.228068 1239 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.45.253:38007: connect: Connection refused (error 111)
04:00:06.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.231971 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.253:38007: Network error: Client connection negotiation failed: client connection to 127.1.45.253:38007: connect: Connection refused (error 111)
04:00:06.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.281478 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } attempt: 1
04:00:06.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.285419 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.253:38007: Network error: Client connection negotiation failed: client connection to 127.1.45.253:38007: connect: Connection refused (error 111)
04:00:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.353865 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } attempt: 2
04:00:06.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.358106 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.253:38007: Network error: Client connection negotiation failed: client connection to 127.1.45.253:38007: connect: Connection refused (error 111)
04:00:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.449566 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } attempt: 3
04:00:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.453430 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.253:38007: Network error: Client connection negotiation failed: client connection to 127.1.45.253:38007: connect: Connection refused (error 111)
04:00:06.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.525256 1289 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:00:06.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.526209 1289 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:00:06.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.527030 1289 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:00:06.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.566227 1289 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:00:06.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.566636 1289 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:00:06.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.566977 1289 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:00:06.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.567318 1289 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:00:06.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.567669 1289 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:00:06.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.567925 1289 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37033/jwks.json
04:00:06.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.596869 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } attempt: 4
04:00:06.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.600698 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.253:38007: Network error: Client connection negotiation failed: client connection to 127.1.45.253:38007: connect: Connection refused (error 111)
04:00:06.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.606575 1289 master_runner.cc:386] Master server non-default flags:
04:00:06.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:06.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.45.212:46757
04:00:06.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:06.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:06.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/data
04:00:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/wal
04:00:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.45.254:33177,127.1.45.253:38007,127.1.45.252:44305
04:00:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.45.253:38007
04:00:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:00:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37033/jwks.json
04:00:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/data/info.pb
04:00:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.45.253
04:00:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:00:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:00:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:00:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:06.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/logs
04:00:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:00:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:00:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:00:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:00:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:00:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:00:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 03:03:38 UTC on ccycloud.quasar-murucv.root.comops.site
04:00:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454978
04:00:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:00:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.607807 1289 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:00:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.610072 1289 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:00:06.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.631160 1298 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.900225 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } attempt: 5
04:00:06.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.905161 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.253:38007: Network error: Client connection negotiation failed: client connection to 127.1.45.253:38007: connect: Connection refused (error 111)
04:00:07.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.452605 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } attempt: 6
04:00:07.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:07.462368 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.253:38007: Network error: Client connection negotiation failed: client connection to 127.1.45.253:38007: connect: Connection refused (error 111)
04:00:07.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.631249 1299 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:07.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:06.633080 1289 server_base.cc:1034] running on GCE node
04:00:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:06.632026 1301 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:07.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.835577 1289 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:00:07.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.838725 1289 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:00:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.840193 1289 hybrid_clock.cc:648] HybridClock initialized: now 1781841607840152 us; error 54 us; skew 500 ppm
04:00:07.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.841145 1289 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:00:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.847771 1289 webserver.cc:460] Webserver started at http://127.1.45.253:34333/ using document root <none> and password file <none>
04:00:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.848863 1289 fs_manager.cc:359] Metadata directory not provided
04:00:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.849104 1289 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:00:07.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.849622 1289 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:00:07.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.854620 1289 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/data/instance:
04:00:07.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef5e5394bf6743de9b041200f2adca95"
04:00:07.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:07 on dist-test-slave-972x"
04:00:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.855919 1289 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/wal/instance:
04:00:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef5e5394bf6743de9b041200f2adca95"
04:00:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:07 on dist-test-slave-972x"
04:00:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.863801 1289 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
04:00:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.869644 1310 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:00:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.870779 1289 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
04:00:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.871129 1289 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/data,/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/wal
04:00:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef5e5394bf6743de9b041200f2adca95"
04:00:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:07 on dist-test-slave-972x"
04:00:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.871508 1289 fs_report.cc:383] FS layout report
04:00:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:00:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/wal
04:00:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/wal
04:00:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/data/data
04:00:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:00:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:00:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:00:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:00:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:00:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:00:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:00:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:00:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:00:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:00:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:00:07.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.917982 1289 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:00:07.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.920421 1289 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:00:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:07.920946 1289 kserver.cc:163] Server-wide thread pool size limit: 3276
04:00:08.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.006664 1289 rpc_server.cc:307] RPC server started. Bound to: 127.1.45.253:38007
04:00:08.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.006775 1361 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.45.253:38007 every 8 connection(s)
04:00:08.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.009579 1289 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/data/info.pb
04:00:08.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.010861 1207 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu as pid 1289
04:00:08.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.011372 1207 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-1/wal/instance
04:00:08.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.014416 1207 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/wal
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/data
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/logs
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/data/info.pb
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:00:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.45.252:44305
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.45.252
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.45.212:46757
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.45.254:33177,127.1.45.253:38007,127.1.45.252:44305
04:00:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:08.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37033/jwks.json with env {}
04:00:08.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.016440 1362 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:00:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.028719 1362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } has no permanent_uuid. Determining permanent_uuid...
04:00:08.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.056257 1362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } has no permanent_uuid. Determining permanent_uuid...
04:00:08.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.072885 1362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } has no permanent_uuid. Determining permanent_uuid...
04:00:08.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.075381 1313 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:08.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.080224 1362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:08.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.129768 1362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 1
04:00:08.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.133616 1362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.202035 1362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 2
04:00:08.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.207540 1362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:08.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.299036 1362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 3
04:00:08.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.303357 1362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:08.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.347435 1364 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:00:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.347976 1364 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:00:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.348448 1364 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:00:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.379119 1364 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:00:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.379424 1364 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:00:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.379613 1364 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:00:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.379796 1364 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:00:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.380000 1364 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:00:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.380198 1364 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37033/jwks.json
04:00:08.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.414333 1364 master_runner.cc:386] Master server non-default flags:
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.45.212:46757
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/data
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/wal
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.45.254:33177,127.1.45.253:38007,127.1.45.252:44305
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.45.252:44305
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37033/jwks.json
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/data/info.pb
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.45.252
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:00:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:00:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/logs
04:00:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:00:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:00:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:00:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:00:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:00:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:00:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 03:03:38 UTC on ccycloud.quasar-murucv.root.comops.site
04:00:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454978
04:00:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:00:08.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.415508 1364 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:00:08.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.417526 1364 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:00:08.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.440625 1375 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.446774 1362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 4
04:00:08.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.450572 1362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:08.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.523026 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } attempt: 7
04:00:08.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.550559 1287 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } has no permanent_uuid. Determining permanent_uuid...
04:00:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.571527 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:08.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.630203 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 1
04:00:08.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.636591 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.718253 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 2
04:00:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.724048 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:08.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.750476 1362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 5
04:00:08.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.756430 1362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:08.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.809641 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 3
04:00:08.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.816303 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:08.956923 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 4
04:00:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.962940 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:09.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.246578 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 5
04:00:09.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:09.252341 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.304078 1362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 6
04:00:09.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:09.313963 1362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:09.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.440912 1377 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:09.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:08.443100 1374 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:09.689605 1376 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1248 milliseconds
04:00:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.689713 1364 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:00:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.690958 1364 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:00:09.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.693917 1364 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:00:09.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.695317 1364 hybrid_clock.cc:648] HybridClock initialized: now 1781841609695275 us; error 62 us; skew 500 ppm
04:00:09.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.696111 1364 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:00:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.702430 1364 webserver.cc:460] Webserver started at http://127.1.45.252:45559/ using document root <none> and password file <none>
04:00:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.703356 1364 fs_manager.cc:359] Metadata directory not provided
04:00:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.703532 1364 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:00:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.703904 1364 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:00:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.710892 1364 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/data/instance:
04:00:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe"
04:00:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:09 on dist-test-slave-972x"
04:00:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.712355 1364 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/wal/instance:
04:00:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe"
04:00:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:09 on dist-test-slave-972x"
04:00:09.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.721325 1364 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
04:00:09.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.727000 1388 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:00:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.728163 1364 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
04:00:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.728456 1364 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/data,/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/wal
04:00:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe"
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:09 on dist-test-slave-972x"
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.728799 1364 fs_report.cc:383] FS layout report
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/wal
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/wal
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/data/data
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:00:09.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:00:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.783578 1364 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:00:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.785331 1364 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:00:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.785717 1364 kserver.cc:163] Server-wide thread pool size limit: 3276
04:00:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.804973 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 6
04:00:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:09.813148 1287 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.45.252:44305: Network error: Client connection negotiation failed: client connection to 127.1.45.252:44305: connect: Connection refused (error 111)
04:00:09.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.866015 1364 rpc_server.cc:307] RPC server started. Bound to: 127.1.45.252:44305
04:00:09.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.866144 1441 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.45.252:44305 every 8 connection(s)
04:00:09.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.868942 1364 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/data/info.pb
04:00:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.875803 1442 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:00:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.878823 1207 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu as pid 1364
04:00:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.879293 1207 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/master-2/wal/instance
04:00:09.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.892820 1442 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } has no permanent_uuid. Determining permanent_uuid...
04:00:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.918303 1442 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } has no permanent_uuid. Determining permanent_uuid...
04:00:09.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.930349 1442 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } has no permanent_uuid. Determining permanent_uuid...
04:00:09.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.948448 1442 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe: Bootstrap starting.
04:00:09.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.954988 1442 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe: Neither blocks nor log segments found. Creating new log.
04:00:09.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.957367 1442 log.cc:824] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe: Log is configured to *not* fsync() on all Append() calls
04:00:09.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.962750 1442 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe: No bootstrap required, opened a new log
04:00:09.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.984967 1442 raft_consensus.cc:348] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } }
04:00:09.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.985630 1442 raft_consensus.cc:374] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:00:09.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.985888 1442 raft_consensus.cc:729] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f5ade3f8d9e84ab486d11dbe0e597ebe, State: Initialized, Role: FOLLOWER
04:00:09.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.986801 1442 consensus_queue.cc:260] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } }
04:00:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.989478 1451 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } } }
04:00:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.990159 1451 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [sys.catalog]: This master's current role is: FOLLOWER
04:00:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:09.991321 1442 sys_catalog.cc:564] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [sys.catalog]: configured and running, proceeding with master startup.
04:00:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:10.024308 1462 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:10.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:10.025149 1462 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:10.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:10.041085 1394 tablet.cc:2369] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:00:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.016714 1364 master_runner.cc:186] Error getting master registration for 127.1.45.254:33177: OK, instance_id { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" instance_seqno: 1781841606146178 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
04:00:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.051239 1364 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:00:10.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.124219 1451 raft_consensus.cc:482] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:00:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.124753 1451 raft_consensus.cc:504] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } }
04:00:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.126859 1451 leader_election.cc:288] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8772b946b6e74e4abf5180ef7ccce19b (127.1.45.254:33177), ef5e5394bf6743de9b041200f2adca95 (127.1.45.253:38007)
04:00:10.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.374583 1362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 7
04:00:10.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.398993 1362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95: Bootstrap starting.
04:00:10.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.406373 1362 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95: Neither blocks nor log segments found. Creating new log.
04:00:10.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.408654 1362 log.cc:824] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95: Log is configured to *not* fsync() on all Append() calls
04:00:10.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.414629 1362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95: No bootstrap required, opened a new log
04:00:10.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.449255 1362 raft_consensus.cc:348] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } }
04:00:10.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.450124 1362 raft_consensus.cc:374] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:00:10.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.450412 1362 raft_consensus.cc:729] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef5e5394bf6743de9b041200f2adca95, State: Initialized, Role: FOLLOWER
04:00:10.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.451400 1362 consensus_queue.cc:260] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } }
04:00:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.456290 1466 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } } }
04:00:10.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.457110 1466 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [sys.catalog]: This master's current role is: FOLLOWER
04:00:10.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.459000 1362 sys_catalog.cc:564] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [sys.catalog]: configured and running, proceeding with master startup.
04:00:10.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.459160 1337 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef5e5394bf6743de9b041200f2adca95" is_pre_election: true
04:00:10.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.460210 1337 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f5ade3f8d9e84ab486d11dbe0e597ebe in term 0.
04:00:10.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.462069 1392 leader_election.cc:302] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ef5e5394bf6743de9b041200f2adca95, f5ade3f8d9e84ab486d11dbe0e597ebe; no voters:
04:00:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.463331 1451 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [term 0 FOLLOWER]: Leader pre-election won for term 1
04:00:10.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.463691 1451 raft_consensus.cc:482] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
04:00:10.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.463994 1451 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [term 0 FOLLOWER]: Advancing to term 1
04:00:10.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.471184 1451 raft_consensus.cc:504] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } }
04:00:10.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.474186 1337 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef5e5394bf6743de9b041200f2adca95"
04:00:10.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.474776 1337 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [term 0 FOLLOWER]: Advancing to term 1
04:00:10.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.481694 1337 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f5ade3f8d9e84ab486d11dbe0e597ebe in term 1.
04:00:10.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.482605 1451 leader_election.cc:288] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [CANDIDATE]: Term 1 election: Requested vote from peers 8772b946b6e74e4abf5180ef7ccce19b (127.1.45.254:33177), ef5e5394bf6743de9b041200f2adca95 (127.1.45.253:38007)
04:00:10.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.482719 1392 leader_election.cc:302] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ef5e5394bf6743de9b041200f2adca95, f5ade3f8d9e84ab486d11dbe0e597ebe; no voters:
04:00:10.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.483359 1451 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [term 1 FOLLOWER]: Leader election won for term 1
04:00:10.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.485566 1451 raft_consensus.cc:686] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [term 1 LEADER]: Becoming Leader. State: Replica: f5ade3f8d9e84ab486d11dbe0e597ebe, State: Running, Role: LEADER
04:00:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.494246 1451 consensus_queue.cc:237] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } }
04:00:10.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.511376 1469 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [sys.catalog]: SysCatalogTable state changed. Reason: New leader f5ade3f8d9e84ab486d11dbe0e597ebe. Latest consensus state: current_term: 1 leader_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } } }
04:00:10.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.512181 1469 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [sys.catalog]: This master's current role is: LEADER
04:00:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.515576 1473 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
04:00:10.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.523852 1473 catalog_manager.cc:1484] Initializing Kudu cluster ID...
04:00:10.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.544942 1337 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [term 1 FOLLOWER]: Refusing update from remote peer f5ade3f8d9e84ab486d11dbe0e597ebe: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
04:00:10.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.548800 1469 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
04:00:10.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.555467 1481 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
04:00:10.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.594794 1466 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f5ade3f8d9e84ab486d11dbe0e597ebe. Latest consensus state: current_term: 1 leader_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } } }
04:00:10.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.595516 1466 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [sys.catalog]: This master's current role is: FOLLOWER
04:00:10.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.590916 1469 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } } }
04:00:10.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.600184 1469 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [sys.catalog]: This master's current role is: LEADER
04:00:10.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.559674 1289 master_runner.cc:186] Error getting master registration for 127.1.45.254:33177: OK, instance_id { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" instance_seqno: 1781841606146178 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
04:00:10.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.601634 1473 catalog_manager.cc:1347] Generated new cluster ID: a714af6b36ac45b19911aee8c21472ae
04:00:10.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.601802 1289 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:00:10.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.602015 1473 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
04:00:10.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.605360 1485 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } } }
04:00:10.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.606284 1485 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95 [sys.catalog]: This master's current role is: FOLLOWER
04:00:10.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.623342 1473 catalog_manager.cc:1370] Generated new certificate authority record
04:00:10.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.627071 1473 catalog_manager.cc:1504] Loading token signing keys...
04:00:10.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.647141 1473 catalog_manager.cc:5868] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe: Generated new TSK 0
04:00:10.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.649241 1473 catalog_manager.cc:1514] Initializing in-progress tserver states...
04:00:10.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.680457 1207 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/wal
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/data
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/logs
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/data/info.pb
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
04:00:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:00:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.45.193:0
04:00:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.45.193
04:00:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.45.193
04:00:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.45.254:33177,127.1.45.253:38007,127.1.45.252:44305
04:00:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.45.212:46757
04:00:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:10.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
04:00:10.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.846578 1287 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } attempt: 7
04:00:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.879562 1287 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b: Bootstrap starting.
04:00:10.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.894867 1287 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b: Neither blocks nor log segments found. Creating new log.
04:00:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.902077 1287 log.cc:824] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b: Log is configured to *not* fsync() on all Append() calls
04:00:10.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.920256 1287 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b: No bootstrap required, opened a new log
04:00:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.953492 1287 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } }
04:00:10.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.954735 1287 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:00:10.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.955183 1287 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8772b946b6e74e4abf5180ef7ccce19b, State: Initialized, Role: FOLLOWER
04:00:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.956442 1287 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } }
04:00:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.965628 1490 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } } }
04:00:10.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.970458 1490 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b [sys.catalog]: This master's current role is: FOLLOWER
04:00:10.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.971115 1260 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b [term 0 FOLLOWER]: Advancing to term 1
04:00:10.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.970072 1287 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b [sys.catalog]: configured and running, proceeding with master startup.
04:00:10.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.975112 1262 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8772b946b6e74e4abf5180ef7ccce19b" is_pre_election: true
04:00:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.978088 1261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8772b946b6e74e4abf5180ef7ccce19b"
04:00:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.984382 1260 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b [term 1 FOLLOWER]: Refusing update from remote peer f5ade3f8d9e84ab486d11dbe0e597ebe: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
04:00:10.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:10.986287 1469 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [LEADER]: Connected to new peer: Peer: permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
04:00:11.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.063544 1490 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b [sys.catalog]: SysCatalogTable state changed. Reason: New leader f5ade3f8d9e84ab486d11dbe0e597ebe. Latest consensus state: current_term: 1 leader_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } } }
04:00:11.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.067332 1490 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b [sys.catalog]: This master's current role is: FOLLOWER
04:00:11.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.068572 1500 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } } }
04:00:11.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.069558 1500 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b [sys.catalog]: This master's current role is: FOLLOWER
04:00:11.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.102144 1507 catalog_manager.cc:1259] Loaded cluster ID: a714af6b36ac45b19911aee8c21472ae
04:00:11.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.102417 1507 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b: loading cluster ID for follower catalog manager: success
04:00:11.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.110368 1507 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b: acquiring CA information for follower catalog manager: success
04:00:11.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.116681 1507 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 8772b946b6e74e4abf5180ef7ccce19b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
04:00:11.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:11.180454 1487 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:00:11.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:11.181386 1487 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:00:11.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:11.182302 1487 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:00:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:11.237047 1487 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:00:11.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:11.238675 1487 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.45.193
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.275863 1487 tablet_server_runner.cc:78] Tablet server non-default flags:
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.45.212:46757
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/data
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/wal
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.45.193:0
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/data/info.pb
04:00:11.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.45.193
04:00:11.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:11.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.45.254:33177,127.1.45.253:38007,127.1.45.252:44305
04:00:11.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:00:11.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:11.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:00:11.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.45.193
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/logs
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 03:03:38 UTC on ccycloud.quasar-murucv.root.comops.site
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454978
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:00:11.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.277397 1487 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:00:11.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.279328 1487 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:00:11.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:11.304584 1514 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.586659 1481 catalog_manager.cc:1259] Loaded cluster ID: a714af6b36ac45b19911aee8c21472ae
04:00:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.586946 1481 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95: loading cluster ID for follower catalog manager: success
04:00:11.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.602411 1481 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95: acquiring CA information for follower catalog manager: success
04:00:11.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.616061 1481 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ef5e5394bf6743de9b041200f2adca95: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
04:00:11.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.646095 1521 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8772b946b6e74e4abf5180ef7ccce19b" member_type: VOTER last_known_addr { host: "127.1.45.254" port: 33177 } } peers { permanent_uuid: "ef5e5394bf6743de9b041200f2adca95" member_type: VOTER last_known_addr { host: "127.1.45.253" port: 38007 } } peers { permanent_uuid: "f5ade3f8d9e84ab486d11dbe0e597ebe" member_type: VOTER last_known_addr { host: "127.1.45.252" port: 44305 } } }
04:00:11.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:11.650276 1521 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5ade3f8d9e84ab486d11dbe0e597ebe [sys.catalog]: This master's current role is: LEADER
04:00:14.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:11.307415 1515 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:14.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:12.696152 1513 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 1487
04:00:14.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:14.278954 1487 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.970s user 0.656s sys 1.186s
04:00:14.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:14.282124 1487 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.974s user 0.656s sys 1.187s
04:00:14.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:14.282377 1518 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:14.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.283285 1487 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:00:14.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:14.282768 1516 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2974 milliseconds
04:00:14.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.287902 1487 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:00:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.290166 1487 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:00:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.291615 1487 hybrid_clock.cc:648] HybridClock initialized: now 1781841614291539 us; error 51 us; skew 500 ppm
04:00:14.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.292762 1487 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:00:14.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.305258 1487 webserver.cc:460] Webserver started at http://127.1.45.193:44407/ using document root <none> and password file <none>
04:00:14.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.306181 1487 fs_manager.cc:359] Metadata directory not provided
04:00:14.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.306382 1487 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:00:14.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.307008 1487 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:00:14.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.311882 1487 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/data/instance:
04:00:14.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2cbc80f3d95421189a80c6061e984b0"
04:00:14.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:14 on dist-test-slave-972x"
04:00:14.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.313002 1487 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/wal/instance:
04:00:14.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2cbc80f3d95421189a80c6061e984b0"
04:00:14.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:14 on dist-test-slave-972x"
04:00:14.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.320647 1487 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.005s
04:00:14.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.327006 1534 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:00:14.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.328240 1487 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
04:00:14.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.328516 1487 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/data,/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/wal
04:00:14.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2cbc80f3d95421189a80c6061e984b0"
04:00:14.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:14 on dist-test-slave-972x"
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.328809 1487 fs_report.cc:383] FS layout report
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/wal
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/wal
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/data/data
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:00:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:00:14.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.368551 1487 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:00:14.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.370105 1487 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:00:14.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.370555 1487 kserver.cc:163] Server-wide thread pool size limit: 3276
04:00:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.373639 1487 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
04:00:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.377950 1487 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
04:00:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.378183 1487 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
04:00:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.378484 1487 ts_tablet_manager.cc:610] Registered 0 tablets
04:00:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.378722 1487 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
04:00:14.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.568831 1487 rpc_server.cc:307] RPC server started. Bound to: 127.1.45.193:36395
04:00:14.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.568929 1646 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.45.193:36395 every 8 connection(s)
04:00:14.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.571394 1487 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/data/info.pb
04:00:14.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.576520 1207 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu as pid 1487
04:00:14.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.577133 1207 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-0/wal/instance
04:00:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.591771 1207 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/wal
04:00:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/data
04:00:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:00:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:00:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:00:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/logs
04:00:14.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/data/info.pb
04:00:14.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:14.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:00:14.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:00:14.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:00:14.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:00:14.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:14.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:14.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:14.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
04:00:14.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:00:14.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.45.194:0
04:00:14.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.45.194
04:00:14.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.45.194
04:00:14.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:14.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.45.254:33177,127.1.45.253:38007,127.1.45.252:44305
04:00:14.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.45.212:46757
04:00:14.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:14.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:14.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
04:00:14.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.642295 1647 heartbeater.cc:347] Connected to a master server at 127.1.45.252:44305
04:00:14.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.642931 1647 heartbeater.cc:464] Registering TS with master...
04:00:14.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.644871 1647 heartbeater.cc:511] Master 127.1.45.252:44305 requested a full tablet report, sending...
04:00:14.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.645880 1648 heartbeater.cc:347] Connected to a master server at 127.1.45.253:38007
04:00:14.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.646217 1648 heartbeater.cc:464] Registering TS with master...
04:00:14.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.647408 1648 heartbeater.cc:511] Master 127.1.45.253:38007 requested a full tablet report, sending...
04:00:14.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.648178 1407 ts_manager.cc:194] Registered new tserver with Master: b2cbc80f3d95421189a80c6061e984b0 (127.1.45.193:36395)
04:00:14.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.652031 1327 ts_manager.cc:194] Registered new tserver with Master: b2cbc80f3d95421189a80c6061e984b0 (127.1.45.193:36395)
04:00:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.654106 1407 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.45.193:55417
04:00:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.654502 1651 heartbeater.cc:347] Connected to a master server at 127.1.45.254:33177
04:00:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.655190 1651 heartbeater.cc:464] Registering TS with master...
04:00:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.655992 1651 heartbeater.cc:511] Master 127.1.45.254:33177 requested a full tablet report, sending...
04:00:14.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:14.659193 1252 ts_manager.cc:194] Registered new tserver with Master: b2cbc80f3d95421189a80c6061e984b0 (127.1.45.193:36395)
04:00:14.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:14.947954 1654 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:00:14.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:14.948552 1654 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:00:14.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:14.949146 1654 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:00:14.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:14.981653 1654 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:00:14.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:14.982661 1654 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.45.194
04:00:15.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:15.018464 1654 tablet_server_runner.cc:78] Tablet server non-default flags:
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.45.212:46757
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/data
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/wal
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.45.194:0
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/data/info.pb
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.45.194
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.45.254:33177,127.1.45.253:38007,127.1.45.252:44305
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.45.194
04:00:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/logs
04:00:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:00:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:00:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
04:00:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:00:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:00:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:00:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 03:03:38 UTC on ccycloud.quasar-murucv.root.comops.site
04:00:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454978
04:00:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:00:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:15.020046 1654 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:00:15.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:15.022058 1654 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:00:15.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:15.042191 1665 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:15.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:15.659560 1647 heartbeater.cc:503] Master 127.1.45.252:44305 was elected leader, sending a full tablet report...
04:00:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:15.042291 1666 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:15.042202 1668 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:16.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:15.044028 1654 server_base.cc:1034] running on GCE node
04:00:16.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.262007 1654 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:00:16.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.264698 1654 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:00:16.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.266131 1654 hybrid_clock.cc:648] HybridClock initialized: now 1781841616266090 us; error 51 us; skew 500 ppm
04:00:16.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.267035 1654 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:00:16.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.273876 1654 webserver.cc:460] Webserver started at http://127.1.45.194:46175/ using document root <none> and password file <none>
04:00:16.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.274917 1654 fs_manager.cc:359] Metadata directory not provided
04:00:16.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.275153 1654 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:00:16.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.275668 1654 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:00:16.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.280453 1654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/data/instance:
04:00:16.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca9bd85b626f43eeb4dec1fa18d6efaa"
04:00:16.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:16 on dist-test-slave-972x"
04:00:16.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.281623 1654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/wal/instance:
04:00:16.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca9bd85b626f43eeb4dec1fa18d6efaa"
04:00:16.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:16 on dist-test-slave-972x"
04:00:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.289026 1654 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
04:00:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.295332 1675 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:00:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.296435 1654 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.296701 1654 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/data,/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/wal
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca9bd85b626f43eeb4dec1fa18d6efaa"
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:16 on dist-test-slave-972x"
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.297003 1654 fs_report.cc:383] FS layout report
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/wal
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/wal
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/data/data
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:00:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:00:16.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:00:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.338472 1654 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:00:16.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.339974 1654 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:00:16.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.340376 1654 kserver.cc:163] Server-wide thread pool size limit: 3276
04:00:16.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.342922 1654 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
04:00:16.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.347538 1654 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
04:00:16.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.347805 1654 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
04:00:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.348026 1654 ts_tablet_manager.cc:610] Registered 0 tablets
04:00:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.348155 1654 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
04:00:16.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.500155 1654 rpc_server.cc:307] RPC server started. Bound to: 127.1.45.194:39593
04:00:16.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.500296 1788 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.45.194:39593 every 8 connection(s)
04:00:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.503019 1654 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/data/info.pb
04:00:16.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.505690 1207 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu as pid 1654
04:00:16.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.506271 1207 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-1/wal/instance
04:00:16.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.520025 1207 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:16.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:16.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/wal
04:00:16.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/data
04:00:16.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:00:16.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:00:16.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:00:16.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:16.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/logs
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/data/info.pb
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.45.195:0
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.45.195
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.45.195
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.45.254:33177,127.1.45.253:38007,127.1.45.252:44305
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.45.212:46757
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:16.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
04:00:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.580067 1789 heartbeater.cc:347] Connected to a master server at 127.1.45.252:44305
04:00:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.580566 1789 heartbeater.cc:464] Registering TS with master...
04:00:16.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.581543 1789 heartbeater.cc:511] Master 127.1.45.252:44305 requested a full tablet report, sending...
04:00:16.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.582696 1794 heartbeater.cc:347] Connected to a master server at 127.1.45.254:33177
04:00:16.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.583021 1794 heartbeater.cc:464] Registering TS with master...
04:00:16.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.583722 1794 heartbeater.cc:511] Master 127.1.45.254:33177 requested a full tablet report, sending...
04:00:16.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.585623 1407 ts_manager.cc:194] Registered new tserver with Master: ca9bd85b626f43eeb4dec1fa18d6efaa (127.1.45.194:39593)
04:00:16.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.586648 1252 ts_manager.cc:194] Registered new tserver with Master: ca9bd85b626f43eeb4dec1fa18d6efaa (127.1.45.194:39593)
04:00:16.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.587116 1407 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.45.194:54453
04:00:16.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.587723 1790 heartbeater.cc:347] Connected to a master server at 127.1.45.253:38007
04:00:16.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.588047 1790 heartbeater.cc:464] Registering TS with master...
04:00:16.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.588848 1790 heartbeater.cc:511] Master 127.1.45.253:38007 requested a full tablet report, sending...
04:00:16.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.591473 1327 ts_manager.cc:194] Registered new tserver with Master: ca9bd85b626f43eeb4dec1fa18d6efaa (127.1.45.194:39593)
04:00:16.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:16.859731 1795 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:00:16.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:16.860342 1795 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:00:16.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:16.860837 1795 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:00:16.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:16.908449 1795 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:00:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:16.909248 1795 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.45.195
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.943112 1795 tablet_server_runner.cc:78] Tablet server non-default flags:
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.45.212:46757
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/data
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/wal
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.45.195:0
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/data/info.pb
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.45.195
04:00:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.45.254:33177,127.1.45.253:38007,127.1.45.252:44305
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.45.195
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/logs
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 03:03:38 UTC on ccycloud.quasar-murucv.root.comops.site
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454978
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:00:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.944348 1795 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:00:16.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.945863 1795 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:00:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:16.967661 1805 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:17.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:17.591115 1789 heartbeater.cc:503] Master 127.1.45.252:44305 was elected leader, sending a full tablet report...
04:00:18.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:16.968171 1806 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:18.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:16.971127 1795 server_base.cc:1034] running on GCE node
04:00:18.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:16.968704 1808 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:18.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.195916 1795 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:00:18.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.200783 1795 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:00:18.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.202236 1795 hybrid_clock.cc:648] HybridClock initialized: now 1781841618202181 us; error 73 us; skew 500 ppm
04:00:18.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.203583 1795 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:00:18.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.211931 1795 webserver.cc:460] Webserver started at http://127.1.45.195:36281/ using document root <none> and password file <none>
04:00:18.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.212895 1795 fs_manager.cc:359] Metadata directory not provided
04:00:18.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.213099 1795 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:00:18.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.213501 1795 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:00:18.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.219409 1795 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/data/instance:
04:00:18.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9ac5a67955f47eeb3c161ec3860ae50"
04:00:18.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:18 on dist-test-slave-972x"
04:00:18.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.220537 1795 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/wal/instance:
04:00:18.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9ac5a67955f47eeb3c161ec3860ae50"
04:00:18.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:18 on dist-test-slave-972x"
04:00:18.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.228839 1795 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
04:00:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.234839 1815 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:00:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.235917 1795 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
04:00:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.236275 1795 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/data,/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/wal
04:00:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c9ac5a67955f47eeb3c161ec3860ae50"
04:00:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:18 on dist-test-slave-972x"
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.236647 1795 fs_report.cc:383] FS layout report
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/wal
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/wal
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/data/data
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:00:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:00:18.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.284792 1795 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:00:18.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.286634 1795 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:00:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.287164 1795 kserver.cc:163] Server-wide thread pool size limit: 3276
04:00:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.289980 1795 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
04:00:18.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.294456 1795 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
04:00:18.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.294705 1795 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
04:00:18.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.295053 1795 ts_tablet_manager.cc:610] Registered 0 tablets
04:00:18.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.295238 1795 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
04:00:18.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.439366 1795 rpc_server.cc:307] RPC server started. Bound to: 127.1.45.195:39251
04:00:18.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.439476 1928 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.45.195:39251 every 8 connection(s)
04:00:18.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.442603 1795 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/data/info.pb
04:00:18.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.449174 1207 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu as pid 1795
04:00:18.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.449810 1207 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster2291993768058875854/ts-2/wal/instance
04:00:18.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.500155 1930 heartbeater.cc:347] Connected to a master server at 127.1.45.253:38007
04:00:18.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.500674 1930 heartbeater.cc:464] Registering TS with master...
04:00:18.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.501969 1930 heartbeater.cc:511] Master 127.1.45.253:38007 requested a full tablet report, sending...
04:00:18.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.505487 1327 ts_manager.cc:194] Registered new tserver with Master: c9ac5a67955f47eeb3c161ec3860ae50 (127.1.45.195:39251)
04:00:18.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.506247 1931 heartbeater.cc:347] Connected to a master server at 127.1.45.254:33177
04:00:18.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.506592 1931 heartbeater.cc:464] Registering TS with master...
04:00:18.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.507807 1931 heartbeater.cc:511] Master 127.1.45.254:33177 requested a full tablet report, sending...
04:00:18.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.510320 1252 ts_manager.cc:194] Registered new tserver with Master: c9ac5a67955f47eeb3c161ec3860ae50 (127.1.45.195:39251)
04:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.519158 1929 heartbeater.cc:347] Connected to a master server at 127.1.45.252:44305
04:00:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.519447 1929 heartbeater.cc:464] Registering TS with master...
04:00:18.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.520083 1929 heartbeater.cc:511] Master 127.1.45.252:44305 requested a full tablet report, sending...
04:00:18.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.522531 1407 ts_manager.cc:194] Registered new tserver with Master: c9ac5a67955f47eeb3c161ec3860ae50 (127.1.45.195:39251)
04:00:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.524591 1407 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.45.195:35729
04:00:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:18.531989 1207 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
04:00:18.543 [DEBUG - main] (MiniKuduCluster.java:195) Response:
04:00:18.544 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
04:00:18.556 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.45.254"
port: 33177
}
bound_http_address {
host: "127.1.45.254"
port: 40941
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.45.253"
port: 38007
}
bound_http_address {
host: "127.1.45.253"
port: 34333
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.45.252"
port: 44305
}
bound_http_address {
host: "127.1.45.252"
port: 45559
}
}
}
04:00:18.563 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
04:00:18.567 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.1.45.193"
port: 36395
}
bound_http_address {
host: "127.1.45.193"
port: 44407
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.1.45.194"
port: 39593
}
bound_http_address {
host: "127.1.45.194"
port: 46175
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.1.45.195"
port: 39251
}
bound_http_address {
host: "127.1.45.195"
port: 36281
}
}
}
04:00:18.567 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
04:00:18.751 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
04:00:18.873 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskJlmI1R/test-tmp
04:00:18.874 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
04:00:18.874 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskJlmI1R/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
04:00:18.880 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
04:00:19.299 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:00:19.299 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
04:00:19.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:19.303200 1940 mini_oidc.cc:152] Starting JWKS server
04:00:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:19.306666 1940 webserver.cc:460] Webserver started at http://127.0.0.1:44475/ using document root <none> and password file <none>
04:00:19.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:19.307065 1940 mini_oidc.cc:189] Starting OIDC Discovery server
04:00:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:19.309355 1940 webserver.cc:460] Webserver started at http://127.0.0.1:41687/ using document root <none> and password file <none>
04:00:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
04:00:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/principal' for realm 'KRBTEST.COM',
04:00:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
04:00:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:19 dist-test-slave-972x krb5kdc[1952](info): setting up network...
04:00:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
04:00:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:19 dist-test-slave-972x krb5kdc[1952](info): set up 2 sockets
04:00:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:19 dist-test-slave-972x krb5kdc[1952](info): commencing operation
04:00:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
04:00:19.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:19.528738 1929 heartbeater.cc:503] Master 127.1.45.252:44305 was elected leader, sending a full tablet report...
04:00:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:20.530617 1940 mini_kdc.cc:121] Time spent starting KDC: real 1.195s user 0.002s sys 0.018s
04:00:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
04:00:20.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:00:20.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
04:00:20.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
04:00:20.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:00:20.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
04:00:20.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
04:00:20.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:00:20.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
04:00:20.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:00:20.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/test-user.keytab.
04:00:20.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/test-user.keytab.
04:00:20.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:20 dist-test-slave-972x krb5kdc[1952](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841620, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:00:20.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
04:00:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T04:00:20Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
04:00:20.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T04:00:20Z Disabled control of system clock
04:00:20.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.229.62@KRBTEST.COM; defaulting to no policy
04:00:20.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:00:20.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.229.62@KRBTEST.COM" created.
04:00:20.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:00:20.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.229.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.62.keytab.
04:00:20.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.229.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.62.keytab.
04:00:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:20.727291 1940 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal
04:00:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data
04:00:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:00:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:00:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:00:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/logs
04:00:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data/info.pb
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.229.62:41303
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.229.62
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.62.keytab
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.229.62
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.229.20:33641
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44475/jwks.json
04:00:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:00:21.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:21.031123 1965 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:00:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:21.031813 1965 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:00:21.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:21.032546 1965 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:00:21.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:21.063858 1965 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:00:21.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:21.064139 1965 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:00:21.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:21.064528 1965 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:00:21.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:21.064852 1965 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:00:21.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:21.065151 1965 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:00:21.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:21.065449 1965 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:00:21.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:21.065654 1965 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44475/jwks.json
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:21.102891 1965 master_runner.cc:386] Master server non-default flags:
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.229.20:33641
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.62.keytab
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.229.62
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.229.62:41303
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44475/jwks.json
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data/info.pb
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.229.62
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/logs
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 03:03:38 UTC on ccycloud.quasar-murucv.root.comops.site
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454978
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:00:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:21.104406 1965 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:00:21.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:21.105546 1965 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:00:22.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:21.126837 1972 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:22.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:21.127120 1973 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:22.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:21.126875 1975 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:22.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:21.128836 1965 server_base.cc:1034] running on GCE node
04:00:22.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.375412 1965 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:00:22.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.379515 1965 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:00:22.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.380926 1965 hybrid_clock.cc:648] HybridClock initialized: now 1781841622380906 us; error 59 us; skew 500 ppm
04:00:22.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.382091 1965 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:00:22.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:22 dist-test-slave-972x krb5kdc[1952](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841622, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:00:22.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.388797 1965 init.cc:377] Logged in from keytab as kudu/127.1.229.62@KRBTEST.COM (short username kudu)
04:00:22.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.394407 1965 webserver.cc:460] Webserver started at http://127.1.229.62:38821/ using document root <none> and password file <none>
04:00:22.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.395604 1965 fs_manager.cc:359] Metadata directory not provided
04:00:22.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.395843 1965 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:00:22.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.396350 1965 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:00:22.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.401187 1965 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data/instance:
04:00:22.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afd283f610b941d280ee0e72ba536b9c"
04:00:22.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:22 on dist-test-slave-972x"
04:00:22.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.402364 1965 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal/instance:
04:00:22.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afd283f610b941d280ee0e72ba536b9c"
04:00:22.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:22 on dist-test-slave-972x"
04:00:22.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.409924 1965 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
04:00:22.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.415824 1986 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:00:22.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.416957 1965 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
04:00:22.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.417313 1965 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data,/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal
04:00:22.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afd283f610b941d280ee0e72ba536b9c"
04:00:22.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:22 on dist-test-slave-972x"
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.417665 1965 fs_report.cc:383] FS layout report
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data/data
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:00:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:00:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.465611 1965 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:00:22.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.471942 1965 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:00:22.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.472451 1965 kserver.cc:163] Server-wide thread pool size limit: 3276
04:00:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.558928 1965 rpc_server.cc:307] RPC server started. Bound to: 127.1.229.62:41303
04:00:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.559002 2037 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.229.62:41303 every 8 connection(s)
04:00:22.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.561894 1965 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data/info.pb
04:00:22.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.568360 1940 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu as pid 1965
04:00:22.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.568439 2038 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:00:22.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.568867 1940 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal/instance
04:00:22.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.583796 2038 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } has no permanent_uuid. Determining permanent_uuid...
04:00:22.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.229.61@KRBTEST.COM; defaulting to no policy
04:00:22.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:00:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.229.61@KRBTEST.COM" created.
04:00:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:22 dist-test-slave-972x krb5kdc[1952](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841622, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.62@KRBTEST.COM for kudu/127.1.229.62@KRBTEST.COM
04:00:22.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.781045 2038 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } has no permanent_uuid. Determining permanent_uuid...
04:00:22.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:22.783152 1990 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:22.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:22.787653 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.61:46763: Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:22.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.837103 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } attempt: 1
04:00:22.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:22.841033 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.61:46763: Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:22.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:22.909415 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } attempt: 2
04:00:22.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:22.913143 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.61:46763: Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:23.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:23.004485 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } attempt: 3
04:00:23.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:23.008154 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.61:46763: Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:23.151536 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } attempt: 4
04:00:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:23.156204 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.61:46763: Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:23.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:23.455557 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } attempt: 5
04:00:23.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:23.459069 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.61:46763: Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:23.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:00:23.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.229.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.61.keytab.
04:00:23.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.229.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.61.keytab.
04:00:23.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:23.893198 1940 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.1.229.61: real 1.321s user 0.002s sys 0.009s
04:00:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:23.895404 1940 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal
04:00:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data
04:00:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:00:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:00:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:00:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/logs
04:00:23.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data/info.pb
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.229.61:46763
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.229.61
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.61.keytab
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.229.61
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.229.20:33641
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:23.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:23.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243
04:00:23.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:23.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44475/jwks.json
04:00:23.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:00:24.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:24.006493 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } attempt: 6
04:00:24.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.014178 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.61:46763: Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:24.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.220127 2045 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:00:24.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.220819 2045 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:00:24.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.221576 2045 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:00:24.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.255539 2045 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:00:24.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.255817 2045 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:00:24.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.256201 2045 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:00:24.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.256538 2045 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:00:24.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.256843 2045 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:00:24.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.257149 2045 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:00:24.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.257396 2045 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44475/jwks.json
04:00:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:24.294377 2045 master_runner.cc:386] Master server non-default flags:
04:00:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:24.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.229.20:33641
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.61.keytab
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.229.61
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.229.61:46763
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44475/jwks.json
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data/info.pb
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.229.61
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:00:24.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/logs
04:00:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:00:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:00:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:00:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:00:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:00:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:00:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 03:03:38 UTC on ccycloud.quasar-murucv.root.comops.site
04:00:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454978
04:00:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:00:24.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:24.295908 2045 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:00:24.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:24.297185 2045 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:00:24.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.314716 2054 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:25.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.074599 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } attempt: 7
04:00:25.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:25.081173 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.61:46763: Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:25.572623 1438 debug-util.cc:398] Leaking SignalData structure 0x7b08000b95a0 after lost signal to thread 1369
04:00:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:25.573308 1438 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0380 after lost signal to thread 1441
04:00:25.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.314706 2055 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:25.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:24.316495 2045 server_base.cc:1034] running on GCE node
04:00:25.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:24.314927 2057 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.603971 2045 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:00:25.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.607100 2045 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:00:25.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.608541 2045 hybrid_clock.cc:648] HybridClock initialized: now 1781841625608481 us; error 78 us; skew 500 ppm
04:00:25.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.609432 2045 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:00:25.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:25 dist-test-slave-972x krb5kdc[1952](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841625, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:00:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.623090 2045 init.cc:377] Logged in from keytab as kudu/127.1.229.61@KRBTEST.COM (short username kudu)
04:00:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.629688 2045 webserver.cc:460] Webserver started at http://127.1.229.61:35363/ using document root <none> and password file <none>
04:00:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.630663 2045 fs_manager.cc:359] Metadata directory not provided
04:00:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.630853 2045 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:00:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.631299 2045 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:00:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.638146 2045 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data/instance:
04:00:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a1de6b27361498c8561cb68eb89897c"
04:00:25.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:25 on dist-test-slave-972x"
04:00:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.639266 2045 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal/instance:
04:00:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a1de6b27361498c8561cb68eb89897c"
04:00:25.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:25 on dist-test-slave-972x"
04:00:25.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.647202 2045 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
04:00:25.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.654026 2069 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:00:25.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.655315 2045 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
04:00:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.655699 2045 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data,/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal
04:00:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a1de6b27361498c8561cb68eb89897c"
04:00:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:25 on dist-test-slave-972x"
04:00:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.656091 2045 fs_report.cc:383] FS layout report
04:00:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:00:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal
04:00:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal
04:00:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data/data
04:00:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:00:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:00:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:00:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:00:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:00:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:00:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:00:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:00:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:00:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:00:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:00:25.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.697319 2045 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:00:25.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.704618 2045 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:00:25.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.705590 2045 kserver.cc:163] Server-wide thread pool size limit: 3276
04:00:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.806143 2045 rpc_server.cc:307] RPC server started. Bound to: 127.1.229.61:46763
04:00:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.806317 2120 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.229.61:46763 every 8 connection(s)
04:00:25.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.809020 2045 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data/info.pb
04:00:25.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.811367 1940 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu as pid 2045
04:00:25.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.811962 1940 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal/instance
04:00:25.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.818259 2121 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:00:25.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.229.60@KRBTEST.COM; defaulting to no policy
04:00:25.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:25.838364 2121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } has no permanent_uuid. Determining permanent_uuid...
04:00:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:00:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.229.60@KRBTEST.COM" created.
04:00:26.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:26 dist-test-slave-972x krb5kdc[1952](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841625, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.61@KRBTEST.COM for kudu/127.1.229.62@KRBTEST.COM
04:00:26.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:26.093806 2121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } has no permanent_uuid. Determining permanent_uuid...
04:00:26.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:26 dist-test-slave-972x krb5kdc[1952](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841625, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.61@KRBTEST.COM for kudu/127.1.229.61@KRBTEST.COM
04:00:26.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:00:26.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.229.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.60.keytab.
04:00:26.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.229.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.60.keytab.
04:00:26.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.225769 1940 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.1.229.60: real 0.411s user 0.004s sys 0.012s
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:26.227562 1940 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/wal
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/logs
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data/info.pb
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:26.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.229.60:43243
04:00:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.229.60
04:00:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.60.keytab
04:00:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.229.60
04:00:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:00:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:00:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:00:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.229.20:33641
04:00:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243
04:00:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44475/jwks.json
04:00:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:00:26.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:26.236178 2121 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } has no permanent_uuid. Determining permanent_uuid...
04:00:26.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.239173 2070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:26.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.243669 2121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.60:43243: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:26.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:26.293226 2121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } attempt: 1
04:00:26.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.297271 2121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.60:43243: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:26.365738 2121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } attempt: 2
04:00:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.369822 2121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.60:43243: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:26.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:26.461328 2121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } attempt: 3
04:00:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.465981 2121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.60:43243: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:26.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.575127 2128 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:00:26.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.575866 2128 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:00:26.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.576872 2128 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:00:26.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:26.609472 2121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } attempt: 4
04:00:26.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.613842 2121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.60:43243: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:26.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.614807 2128 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:00:26.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.615008 2128 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:00:26.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.615274 2128 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:00:26.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.615584 2128 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:00:26.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.615878 2128 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:00:26.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.616098 2128 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:00:26.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.616242 2128 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44475/jwks.json
04:00:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:26.658630 2128 master_runner.cc:386] Master server non-default flags:
04:00:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.229.20:33641
04:00:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data
04:00:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/wal
04:00:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:00:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243
04:00:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.60.keytab
04:00:26.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.229.60
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.229.60:43243
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44475/jwks.json
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data/info.pb
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.229.60
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/logs
04:00:26.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:00:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:00:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:00:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:00:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:00:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:00:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 03:03:38 UTC on ccycloud.quasar-murucv.root.comops.site
04:00:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454978
04:00:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:00:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:26.660037 2128 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:00:26.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:26.661199 2128 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:00:26.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.681476 2136 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:26.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:26.913278 2121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } attempt: 5
04:00:26.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.921522 2121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.60:43243: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:27.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:27.171759 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } attempt: 8
04:00:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:27 dist-test-slave-972x krb5kdc[1952](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841622, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.62@KRBTEST.COM for kudu/127.1.229.61@KRBTEST.COM
04:00:27.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:27.269667 2038 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } has no permanent_uuid. Determining permanent_uuid...
04:00:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:27.302079 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.60:43243: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:27.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:27.367816 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } attempt: 1
04:00:27.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:27.376116 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.60:43243: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:27.429822 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } attempt: 2
04:00:27.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:27.453290 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.60:43243: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:27.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:27.469138 2121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } attempt: 6
04:00:27.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:27.507929 2121 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.60:43243: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:27.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:27.530012 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } attempt: 3
04:00:27.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:27.537535 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.60:43243: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:27.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:27.693114 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } attempt: 4
04:00:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:27.707080 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.60:43243: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.003839 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } attempt: 5
04:00:28.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:28.010092 1987 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111) [suppressed 12 similar messages]
04:00:28.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:28.014747 2038 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.229.60:43243: Network error: Client connection negotiation failed: client connection to 127.1.229.60:43243: connect: Connection refused (error 111)
04:00:28.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.682384 2137 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:28.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:26.685635 2139 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:28.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:28.116037 2138 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1432 milliseconds
04:00:28.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.116158 2128 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:00:28.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.117545 2128 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:00:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.120085 2128 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:00:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.121444 2128 hybrid_clock.cc:648] HybridClock initialized: now 1781841628121413 us; error 43 us; skew 500 ppm
04:00:28.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.122316 2128 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:00:28.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:28 dist-test-slave-972x krb5kdc[1952](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841628, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:00:28.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.127943 2128 init.cc:377] Logged in from keytab as kudu/127.1.229.60@KRBTEST.COM (short username kudu)
04:00:28.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.133764 2128 webserver.cc:460] Webserver started at http://127.1.229.60:35603/ using document root <none> and password file <none>
04:00:28.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.134961 2128 fs_manager.cc:359] Metadata directory not provided
04:00:28.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.135205 2128 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:00:28.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.135589 2128 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:00:28.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.140415 2128 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data/instance:
04:00:28.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25d212d26cb54c6db7ea1fe7eb95494b"
04:00:28.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:28 on dist-test-slave-972x"
04:00:28.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.141443 2128 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/wal/instance:
04:00:28.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25d212d26cb54c6db7ea1fe7eb95494b"
04:00:28.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:28 on dist-test-slave-972x"
04:00:28.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.149072 2128 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
04:00:28.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.154948 2154 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:00:28.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.156100 2128 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
04:00:28.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.156380 2128 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data,/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/wal
04:00:28.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25d212d26cb54c6db7ea1fe7eb95494b"
04:00:28.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:28 on dist-test-slave-972x"
04:00:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.156767 2128 fs_report.cc:383] FS layout report
04:00:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:00:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/wal
04:00:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/wal
04:00:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data/data
04:00:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:00:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:00:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:00:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:00:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:00:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:00:28.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:00:28.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:00:28.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:00:28.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:00:28.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:00:28.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.226894 2128 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:00:28.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.232366 2128 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:00:28.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.232890 2128 kserver.cc:163] Server-wide thread pool size limit: 3276
04:00:28.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.312364 2128 rpc_server.cc:307] RPC server started. Bound to: 127.1.229.60:43243
04:00:28.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.312446 2205 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.229.60:43243 every 8 connection(s)
04:00:28.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.315163 2128 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data/info.pb
04:00:28.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.320533 1940 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu as pid 2128
04:00:28.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.320818 2206 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:00:28.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.320977 1940 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/wal/instance
04:00:28.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.336129 2206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } has no permanent_uuid. Determining permanent_uuid...
04:00:28.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:28 dist-test-slave-972x krb5kdc[1952](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841628, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.60@KRBTEST.COM for kudu/127.1.229.62@KRBTEST.COM
04:00:28.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:28 dist-test-slave-972x krb5kdc[1952](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841620, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.229.60@KRBTEST.COM
04:00:28.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.377485 2206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } has no permanent_uuid. Determining permanent_uuid...
04:00:28.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:28 dist-test-slave-972x krb5kdc[1952](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841628, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.60@KRBTEST.COM for kudu/127.1.229.61@KRBTEST.COM
04:00:28.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:28 dist-test-slave-972x krb5kdc[1952](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841620, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.229.61@KRBTEST.COM
04:00:28.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.404031 2206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } has no permanent_uuid. Determining permanent_uuid...
04:00:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:28 dist-test-slave-972x krb5kdc[1952](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841628, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.60@KRBTEST.COM for kudu/127.1.229.60@KRBTEST.COM
04:00:28.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:28 dist-test-slave-972x krb5kdc[1952](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841620, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.229.62@KRBTEST.COM
04:00:28.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.434427 2206 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: Bootstrap starting.
04:00:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.435729 1940 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
04:00:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.440508 2206 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: Neither blocks nor log segments found. Creating new log.
04:00:28.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.442283 2206 log.cc:824] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: Log is configured to *not* fsync() on all Append() calls
04:00:28.445 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:00:28.447 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
04:00:28.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.448137 2206 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: No bootstrap required, opened a new log
04:00:28.457 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
04:00:28.458 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
04:00:28.462 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.229.62"
port: 41303
}
bound_http_address {
host: "127.1.229.62"
port: 38821
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.229.61"
port: 46763
}
bound_http_address {
host: "127.1.229.61"
port: 35363
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.229.60"
port: 43243
}
bound_http_address {
host: "127.1.229.60"
port: 35603
}
}
}
04:00:28.462 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
04:00:28.462 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:00:28.463 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
04:00:28.464 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
04:00:28.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.478330 2206 raft_consensus.cc:348] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } } peers { permanent_uuid: "3a1de6b27361498c8561cb68eb89897c" member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } } peers { permanent_uuid: "25d212d26cb54c6db7ea1fe7eb95494b" member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } }
04:00:28.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.479001 2206 raft_consensus.cc:374] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:00:28.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.479285 2206 raft_consensus.cc:729] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 25d212d26cb54c6db7ea1fe7eb95494b, State: Initialized, Role: FOLLOWER
04:00:28.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.480120 2206 consensus_queue.cc:260] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } } peers { permanent_uuid: "3a1de6b27361498c8561cb68eb89897c" member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } } peers { permanent_uuid: "25d212d26cb54c6db7ea1fe7eb95494b" member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } }
04:00:28.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.483139 2217 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } } peers { permanent_uuid: "3a1de6b27361498c8561cb68eb89897c" member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } } peers { permanent_uuid: "25d212d26cb54c6db7ea1fe7eb95494b" member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } } }
04:00:28.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.484504 2217 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [sys.catalog]: This master's current role is: FOLLOWER
04:00:28.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.484813 2206 sys_catalog.cc:564] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [sys.catalog]: configured and running, proceeding with master startup.
04:00:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:28.489264 2160 tablet.cc:2369] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:00:28.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:28 dist-test-slave-972x krb5kdc[1952](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841628, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:00:28.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
04:00:28.503 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:00:28.506 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5cc
04:00:28.514 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
04:00:28.516 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5cc
04:00:28.517 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
04:00:28.522 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
04:00:28.524 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
04:00:28.524 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
04:00:28.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:28.532053 2229 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:28.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:28.532410 2229 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:28.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.536752 2038 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } attempt: 6
04:00:28.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:28 dist-test-slave-972x krb5kdc[1952](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841622, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.62@KRBTEST.COM for kudu/127.1.229.60@KRBTEST.COM
04:00:28.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.523487 2128 master_runner.cc:186] Error getting master registration for 127.1.229.62:41303: OK, instance_id { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" instance_seqno: 1781841622535758 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
04:00:28.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.562420 2128 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:00:28.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:28 dist-test-slave-972x krb5kdc[1952](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841628, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:00:28.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
04:00:28.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.574581 2121 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } attempt: 7
04:00:28.576 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:00:28.577 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5cc
04:00:28.578 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
04:00:28.579 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.229.62:41303
04:00:28.581 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
04:00:28.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.582125 1940 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu with pid 1965
04:00:28.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:28 dist-test-slave-972x krb5kdc[1952](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841625, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.61@KRBTEST.COM for kudu/127.1.229.60@KRBTEST.COM
04:00:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:28.606470 2156 connection.cc:535] server connection from 127.0.0.1:50724 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
04:00:28.610 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:00:28.610 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.229.61:46763
04:00:28.611 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
04:00:28.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.611336 1940 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu with pid 2045
04:00:28.646 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:00:28.646 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.229.60:43243
04:00:28.646 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
04:00:28.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.647116 1940 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu with pid 2128
04:00:28.671 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:00:28.671 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.229.62:41303
04:00:28.673 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
04:00:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:28.675499 1940 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/logs
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data/info.pb
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.229.62:41303
04:00:28.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.229.62
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38821
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.62.keytab
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.229.62
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.229.20:33641
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44475/jwks.json
04:00:28.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:00:28.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:28.988050 2235 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:00:28.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:28.988749 2235 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:00:28.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:28.989496 2235 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:00:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:29.021910 2235 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:00:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:29.022200 2235 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:00:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:29.022680 2235 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:00:29.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:29.023042 2235 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:00:29.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:29.023397 2235 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:00:29.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:29.023729 2235 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:00:29.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:29.023972 2235 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44475/jwks.json
04:00:29.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:29.062469 2235 master_runner.cc:386] Master server non-default flags:
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.229.20:33641
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.62.keytab
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.229.62
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.229.62:41303
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44475/jwks.json
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data/info.pb
04:00:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.229.62
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38821
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/logs
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 03:03:38 UTC on ccycloud.quasar-murucv.root.comops.site
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454978
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:00:29.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:29.064016 2235 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:00:29.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:29.065212 2235 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:00:29.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:29.082363 2241 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:30.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:29.083050 2242 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:29.083990 2244 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:29.085341 2235 server_base.cc:1034] running on GCE node
04:00:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.325755 2235 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:00:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.328764 2235 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:00:30.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.330153 2235 hybrid_clock.cc:648] HybridClock initialized: now 1781841630330089 us; error 115 us; skew 500 ppm
04:00:30.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.331115 2235 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:00:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:30 dist-test-slave-972x krb5kdc[1952](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841630, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:00:30.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.338232 2235 init.cc:377] Logged in from keytab as kudu/127.1.229.62@KRBTEST.COM (short username kudu)
04:00:30.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.344120 2235 webserver.cc:460] Webserver started at http://127.1.229.62:38821/ using document root <none> and password file <none>
04:00:30.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.345023 2235 fs_manager.cc:359] Metadata directory not provided
04:00:30.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.345204 2235 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:00:30.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.353228 2235 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
04:00:30.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.357813 2254 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:00:30.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.359050 2235 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
04:00:30.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.359400 2235 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data,/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal
04:00:30.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afd283f610b941d280ee0e72ba536b9c"
04:00:30.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:22 on dist-test-slave-972x"
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.361755 2235 fs_report.cc:383] FS layout report
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data/data
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:00:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:00:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.396766 2235 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:00:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.403231 2235 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:00:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.403641 2235 kserver.cc:163] Server-wide thread pool size limit: 3276
04:00:30.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.477399 2235 rpc_server.cc:307] RPC server started. Bound to: 127.1.229.62:41303
04:00:30.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.477582 2305 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.229.62:41303 every 8 connection(s)
04:00:30.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.480172 2235 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/data/info.pb
04:00:30.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.490053 1940 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu as pid 2235
04:00:30.491 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:00:30.491 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.229.61:46763
04:00:30.491 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
04:00:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.492918 2306 sys_catalog.cc:263] Verifying existing consensus state
04:00:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.493386 1940 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal
04:00:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data
04:00:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/logs
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data/info.pb
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:00:30.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:30.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:30.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.229.61:46763
04:00:30.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.229.61
04:00:30.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35363
04:00:30.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.61.keytab
04:00:30.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.229.61
04:00:30.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:00:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:00:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:00:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.229.20:33641
04:00:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243
04:00:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44475/jwks.json
04:00:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:00:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260619 04:00:30.493865 2306 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:00:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.494379 2235 master.cc:537] Master@127.1.229.62:41303 shutting down...
04:00:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.559417 2235 master.cc:552] Master@127.1.229.62:41303 shutdown complete.
04:00:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:00:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:30.818202 2308 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:00:30.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:30.818868 2308 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:00:30.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:30.819469 2308 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:00:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:30.852315 2308 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:00:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:30.852545 2308 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:00:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:30.852806 2308 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:00:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:30.853070 2308 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:00:30.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:30.853262 2308 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:00:30.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:30.853524 2308 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:00:30.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:30.853662 2308 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44475/jwks.json
04:00:30.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.893435 2308 master_runner.cc:386] Master server non-default flags:
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.229.20:33641
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.61.keytab
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.229.61
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.229.61:46763
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44475/jwks.json
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:00:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:30.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:30.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data/info.pb
04:00:30.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.229.61
04:00:30.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35363
04:00:30.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:00:30.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:30.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:00:30.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:00:30.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/logs
04:00:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:00:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:00:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:00:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:00:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:00:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:00:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 03:03:38 UTC on ccycloud.quasar-murucv.root.comops.site
04:00:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454978
04:00:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:00:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.894915 2308 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:00:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.896239 2308 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:00:30.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:30.914870 2315 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:30.915042 2314 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:30.917500 2308 server_base.cc:1034] running on GCE node
04:00:32.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:30.915783 2317 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:32.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.147418 2308 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:00:32.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.150023 2308 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:00:32.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.151381 2308 hybrid_clock.cc:648] HybridClock initialized: now 1781841632151339 us; error 57 us; skew 500 ppm
04:00:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.152171 2308 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:00:32.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:32 dist-test-slave-972x krb5kdc[1952](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841632, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:00:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.164399 2308 init.cc:377] Logged in from keytab as kudu/127.1.229.61@KRBTEST.COM (short username kudu)
04:00:32.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.170069 2308 webserver.cc:460] Webserver started at http://127.1.229.61:35363/ using document root <none> and password file <none>
04:00:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.171113 2308 fs_manager.cc:359] Metadata directory not provided
04:00:32.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.171367 2308 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:00:32.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.181308 2308 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.005s sys 0.004s
04:00:32.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.186331 2326 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:00:32.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.187705 2308 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
04:00:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.188102 2308 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data,/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal
04:00:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a1de6b27361498c8561cb68eb89897c"
04:00:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:25 on dist-test-slave-972x"
04:00:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.191489 2308 fs_report.cc:383] FS layout report
04:00:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:00:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal
04:00:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal
04:00:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data/data
04:00:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:00:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:00:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:00:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:00:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:00:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:00:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:00:32.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:00:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:00:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:00:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:00:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.236274 2308 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:00:32.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.242062 2308 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:00:32.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.242458 2308 kserver.cc:163] Server-wide thread pool size limit: 3276
04:00:32.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.312738 2308 rpc_server.cc:307] RPC server started. Bound to: 127.1.229.61:46763
04:00:32.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.312868 2377 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.229.61:46763 every 8 connection(s)
04:00:32.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.315688 2308 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/data/info.pb
04:00:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.319521 1940 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu as pid 2308
04:00:32.320 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:00:32.321 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.229.60:43243
04:00:32.321 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.322922 1940 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/wal
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/logs
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data/info.pb
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:00:32.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.229.60:43243
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.229.60
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35603
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.60.keytab
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.229.60
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.229.20:33641
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44475/jwks.json
04:00:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:00:32.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.329087 2378 sys_catalog.cc:263] Verifying existing consensus state
04:00:32.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260619 04:00:32.329862 2378 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:00:32.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.330307 2308 master.cc:537] Master@127.1.229.61:46763 shutting down...
04:00:32.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.385293 2308 master.cc:552] Master@127.1.229.61:46763 shutdown complete.
04:00:32.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:00:32.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:32.660104 2380 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:00:32.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:32.660915 2380 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:00:32.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:32.661804 2380 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:00:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:32.693921 2380 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:00:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:32.694144 2380 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:00:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:32.694415 2380 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:00:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:32.694626 2380 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:00:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:32.694816 2380 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:00:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:32.695003 2380 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:00:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:32.695140 2380 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44475/jwks.json
04:00:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.733327 2380 master_runner.cc:386] Master server non-default flags:
04:00:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:00:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.229.20:33641
04:00:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:00:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:00:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data
04:00:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/wal
04:00:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:00:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/krb5kdc/kudu_127.1.229.60.keytab
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.229.60
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.229.60:43243
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44475/jwks.json
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data/info.pb
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.229.60
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35603
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/logs
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 03:03:38 UTC on ccycloud.quasar-murucv.root.comops.site
04:00:32.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4454978
04:00:32.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:00:32.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.734895 2380 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:00:32.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.736181 2380 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:00:32.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:32.754573 2387 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:33.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:32.754575 2386 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:33.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:32.756513 2380 server_base.cc:1034] running on GCE node
04:00:33.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:32.755692 2389 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:00:33.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:33.994943 2380 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:00:33.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:33.997574 2380 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:00:33.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:33.998942 2380 hybrid_clock.cc:648] HybridClock initialized: now 1781841633998907 us; error 47 us; skew 500 ppm
04:00:34.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:33.999804 2380 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:00:34.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:33 dist-test-slave-972x krb5kdc[1952](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841633, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:00:34.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.006093 2380 init.cc:377] Logged in from keytab as kudu/127.1.229.60@KRBTEST.COM (short username kudu)
04:00:34.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.011948 2380 webserver.cc:460] Webserver started at http://127.1.229.60:35603/ using document root <none> and password file <none>
04:00:34.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.012849 2380 fs_manager.cc:359] Metadata directory not provided
04:00:34.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.013044 2380 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:00:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.021538 2380 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
04:00:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.026702 2397 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:00:34.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.027846 2380 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
04:00:34.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.028120 2380 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data,/tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/wal
04:00:34.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25d212d26cb54c6db7ea1fe7eb95494b"
04:00:34.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 04:00:28 on dist-test-slave-972x"
04:00:34.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.030305 2380 fs_report.cc:383] FS layout report
04:00:34.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:00:34.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/wal
04:00:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/wal
04:00:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data/data
04:00:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:00:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:00:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:00:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:00:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:00:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:00:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:00:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:00:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:00:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:00:34.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:00:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.064030 2380 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:00:34.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.069715 2380 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:00:34.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.070237 2380 kserver.cc:163] Server-wide thread pool size limit: 3276
04:00:34.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.141139 2380 rpc_server.cc:307] RPC server started. Bound to: 127.1.229.60:43243
04:00:34.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.141258 2448 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.229.60:43243 every 8 connection(s)
04:00:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.144088 2380 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJlmI1R/test-tmp/mini-kudu-cluster7389946348361704798/master-2/data/info.pb
04:00:34.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.154481 1940 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu as pid 2380
04:00:34.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.154755 2449 sys_catalog.cc:263] Verifying existing consensus state
04:00:34.155 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:00:34.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.160293 2449 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: Bootstrap starting.
04:00:34.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.205607 2449 log.cc:824] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: Log is configured to *not* fsync() on all Append() calls
04:00:34.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.214682 2449 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
04:00:34.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.215827 2449 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: Bootstrap complete.
04:00:34.224 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:34.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.251830 2449 raft_consensus.cc:348] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } } peers { permanent_uuid: "3a1de6b27361498c8561cb68eb89897c" member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } } peers { permanent_uuid: "25d212d26cb54c6db7ea1fe7eb95494b" member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } }
04:00:34.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.255246 2449 raft_consensus.cc:374] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:00:34.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.255581 2449 raft_consensus.cc:729] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 25d212d26cb54c6db7ea1fe7eb95494b, State: Initialized, Role: FOLLOWER
04:00:34.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.256650 2449 consensus_queue.cc:260] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } } peers { permanent_uuid: "3a1de6b27361498c8561cb68eb89897c" member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } } peers { permanent_uuid: "25d212d26cb54c6db7ea1fe7eb95494b" member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } }
04:00:34.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.259581 2453 sys_catalog.cc:455] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } } peers { permanent_uuid: "3a1de6b27361498c8561cb68eb89897c" member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } } peers { permanent_uuid: "25d212d26cb54c6db7ea1fe7eb95494b" member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } } }
04:00:34.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.260540 2453 sys_catalog.cc:458] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [sys.catalog]: This master's current role is: FOLLOWER
04:00:34.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.261713 2449 sys_catalog.cc:564] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [sys.catalog]: configured and running, proceeding with master startup.
04:00:34.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:34.293735 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:34.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:34.294086 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:34.302 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:34.315 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.229.60:43243(127.1.229.60:43243)] connecting to peer
04:00:34.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:34.325388 2403 tablet.cc:2369] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:00:34.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.296326 2380 master_runner.cc:186] Error getting master registration for 127.1.229.62:41303: Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111),
04:00:34.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.329082 2380 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:00:34.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.345 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.346 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.346 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.348 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.348 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.350 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.229.60:43243(127.1.229.60:43243)] Successfully connected to peer
04:00:34.428 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
04:00:34.447 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.229.60
04:00:34.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:34 dist-test-slave-972x krb5kdc[1952](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841628, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.229.60@KRBTEST.COM
04:00:34.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.684074 2453 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:00:34.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.684537 2453 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } } peers { permanent_uuid: "3a1de6b27361498c8561cb68eb89897c" member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } } peers { permanent_uuid: "25d212d26cb54c6db7ea1fe7eb95494b" member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } }
04:00:34.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:34.689152 2401 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111)
04:00:34.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.689390 2453 leader_election.cc:288] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303), 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763)
04:00:34.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:34.694758 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303): Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111)
04:00:34.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:34.696702 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763): Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:34.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.697458 2401 leader_election.cc:302] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25d212d26cb54c6db7ea1fe7eb95494b; no voters: 3a1de6b27361498c8561cb68eb89897c, afd283f610b941d280ee0e72ba536b9c
04:00:34.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:34.698410 2453 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:00:34.708 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd5fef68c, L:/127.0.0.1:50738 - R:/127.1.229.60:43243] using SASL/GSSAPI
04:00:34.725 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:34.745 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:34.748 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:34.753 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.753 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.754 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.755 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.756 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.760 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:34.780 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:34.785 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:34.785 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.785 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.786 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.788 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.791 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:34.800 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:34.801 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:34.802 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.802 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.803 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.806 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.807 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.807 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.809 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:34.841 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:34.842 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:34.843 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.845 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.845 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.856 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.856 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.857 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.857 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:34.881 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:34.882 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:34.884 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.884 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.885 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.885 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.886 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.891 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:34.960 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:34.963 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:34.963 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.964 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.966 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.966 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.966 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:34.967 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:34.969 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:35.061 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:35.063 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:35.064 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:35.064 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:35.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:35.066 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:35.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:35.066 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:35.069 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:35.141 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:35.144 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:35.145 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:35.145 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:35.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:35.147 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:35.147 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:35.148 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:35.154 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:35.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:35.297191 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:35.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:35.297497 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:35.380 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:35.381 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:35.382 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:35.383 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:35.383 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:35.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:35.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:35.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:35.388 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:35.400 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:35.401 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:35.402 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:35.402 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:35.402 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:35.403 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:35.404 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:35.404 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:35.406 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:36.240 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:36.241 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:36.242 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:36.242 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:36.242 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:36.245 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:36.245 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:36.245 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:36.247 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:36.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:36.300509 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:36.300920 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:36.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:36.340197 2474 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:00:36.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:36.340579 2474 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } } peers { permanent_uuid: "3a1de6b27361498c8561cb68eb89897c" member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } } peers { permanent_uuid: "25d212d26cb54c6db7ea1fe7eb95494b" member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } }
04:00:36.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:36.342253 2474 leader_election.cc:288] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303), 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763)
04:00:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:36.348978 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303): Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111)
04:00:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:36.349599 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763): Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:36.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:36.349954 2401 leader_election.cc:302] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25d212d26cb54c6db7ea1fe7eb95494b; no voters: 3a1de6b27361498c8561cb68eb89897c, afd283f610b941d280ee0e72ba536b9c
04:00:36.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:36.350648 2474 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:00:37.080 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:37.081 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:37.083 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:37.083 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:37.084 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:37.084 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:37.084 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:37.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:37.085 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:37.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:37.303977 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:37.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:37.304277 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:37.800 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:37.801 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:37.802 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:37.803 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:37.803 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:37.804 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:37.804 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:37.804 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:37.806 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:38.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:38.307137 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:38.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:38.307440 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:39.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:39.310328 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:39.310694 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:39.332046 2380 master_runner.cc:186] Error getting master registration for 127.1.229.62:41303: Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111),
04:00:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:39.332427 2380 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:00:39.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:39.586110 2484 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:00:39.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:39.586565 2484 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } } peers { permanent_uuid: "3a1de6b27361498c8561cb68eb89897c" member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } } peers { permanent_uuid: "25d212d26cb54c6db7ea1fe7eb95494b" member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } }
04:00:39.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:39.588043 2484 leader_election.cc:288] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303), 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763)
04:00:39.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:39.593991 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303): Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111)
04:00:39.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:39.594628 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763): Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:39.594890 2401 leader_election.cc:302] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25d212d26cb54c6db7ea1fe7eb95494b; no voters: 3a1de6b27361498c8561cb68eb89897c, afd283f610b941d280ee0e72ba536b9c
04:00:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:39.595388 2484 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:00:40.180 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:40.181 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:40.182 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:40.182 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:40.182 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:40.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:40.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:40.185 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:40.187 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:40.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:40.313512 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:40.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:40.313809 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:41.316704 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:41.317003 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:42.319811 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:42.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:42.320123 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:42.900 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:42.902 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:42.903 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:42.903 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:42.903 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:42.904 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:42.904 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:42.905 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:42.907 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:43.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:43.323477 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:43.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:43.323850 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:43.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:43.766760 2491 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:00:43.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:43.767289 2491 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } } peers { permanent_uuid: "3a1de6b27361498c8561cb68eb89897c" member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } } peers { permanent_uuid: "25d212d26cb54c6db7ea1fe7eb95494b" member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } }
04:00:43.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:43.768986 2491 leader_election.cc:288] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303), 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763)
04:00:43.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:43.771873 2401 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111) [suppressed 5 similar messages]
04:00:43.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:43.775808 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303): Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111)
04:00:43.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:43.776357 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763): Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:43.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:43.776659 2401 leader_election.cc:302] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25d212d26cb54c6db7ea1fe7eb95494b; no voters: 3a1de6b27361498c8561cb68eb89897c, afd283f610b941d280ee0e72ba536b9c
04:00:43.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:43.777299 2491 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:00:44.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:44 dist-test-slave-972x krb5kdc[1952](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841644, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:00:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:44.010010 2393 init.cc:295] Successfully reacquired a new kerberos TGT
04:00:44.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:44.326506 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:44.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:44.326838 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:44.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:44.335027 2380 master_runner.cc:186] Error getting master registration for 127.1.229.62:41303: Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111),
04:00:44.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:44.335465 2380 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:00:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:45.329563 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:45.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:45.329831 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:46.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:46.332773 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:46.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:46.333031 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:46.620 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:46.622 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:46.622 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:46.622 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:46.623 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:46.624 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:46.624 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:46.624 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:46.627 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:46.700 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:46.701 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:46.702 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:46.703 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:46.703 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:46.703 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:46.703 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:46.704 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:46.707 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:47.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:47.335785 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:47.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:47.336048 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:48.339061 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:48.339356 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:49.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:49.338171 2380 master_runner.cc:186] Error getting master registration for 127.1.229.62:41303: Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111),
04:00:49.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:49.338604 2380 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:00:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:49.342198 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:49.342429 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:49.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:49.958173 2505 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:00:49.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:49.958643 2505 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } } peers { permanent_uuid: "3a1de6b27361498c8561cb68eb89897c" member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } } peers { permanent_uuid: "25d212d26cb54c6db7ea1fe7eb95494b" member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } }
04:00:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:49.960366 2505 leader_election.cc:288] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303), 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763)
04:00:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:49.963157 2401 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111) [suppressed 1 similar messages]
04:00:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:49.965983 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303): Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111)
04:00:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:49.966441 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763): Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:49.966753 2401 leader_election.cc:302] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25d212d26cb54c6db7ea1fe7eb95494b; no voters: 3a1de6b27361498c8561cb68eb89897c, afd283f610b941d280ee0e72ba536b9c
04:00:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:49.967278 2505 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:00:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:50.345104 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:50.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:50.345355 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:50.680 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:50.681 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:50.682 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:50.683 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:50.683 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:50.684 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:50.684 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:50.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:50.687 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:51.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:51.348546 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:51.348860 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:52.351665 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:52.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:52.351959 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:53.355592 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:53.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:53.355963 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:53.880 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:53.882 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:53.883 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:53.883 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:53.883 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:53.884 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:53.885 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:53.885 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:53.887 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:00:54 dist-test-slave-972x krb5kdc[1952](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841654, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:00:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:54.012483 2393 init.cc:295] Successfully reacquired a new kerberos TGT
04:00:54.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:54.341825 2380 master_runner.cc:186] Error getting master registration for 127.1.229.62:41303: Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111),
04:00:54.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:54.342204 2380 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:00:54.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:54.358692 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:54.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:54.358984 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:54.440 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:54.441 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:54.441 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:54.441 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:54.442 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:54.443 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:54.443 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:54.443 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:54.446 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:55.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:55.168483 2515 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:00:55.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:55.168944 2515 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } } peers { permanent_uuid: "3a1de6b27361498c8561cb68eb89897c" member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } } peers { permanent_uuid: "25d212d26cb54c6db7ea1fe7eb95494b" member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } }
04:00:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:55.170555 2515 leader_election.cc:288] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303), 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763)
04:00:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:55.173446 2401 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111) [suppressed 1 similar messages]
04:00:55.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:55.176674 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303): Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111)
04:00:55.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:55.177299 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763): Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:00:55.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:55.177637 2401 leader_election.cc:302] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25d212d26cb54c6db7ea1fe7eb95494b; no voters: 3a1de6b27361498c8561cb68eb89897c, afd283f610b941d280ee0e72ba536b9c
04:00:55.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:55.178445 2515 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:00:55.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:55.361977 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:55.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:55.362222 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:56.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:56.136961 1785 debug-util.cc:398] Leaking SignalData structure 0x7b08000af600 after lost signal to thread 1659
04:00:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:56.138006 1785 debug-util.cc:398] Leaking SignalData structure 0x7b08000afd00 after lost signal to thread 1788
04:00:56.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:56.366433 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:56.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:56.366792 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:57.370976 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:57.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:57.371310 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:57.520 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:57.521 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:57.521 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:57.521 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:57.522 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:57.522 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:57.522 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:57.523 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:57.526 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:58.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:58.374294 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:58.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:58.374569 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:59.240 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:59.241 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:59.241 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:59.241 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:59.242 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:59.242 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:59.243 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:59.243 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:59.245 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:00:59.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:59.344751 2380 master_runner.cc:186] Error getting master registration for 127.1.229.62:41303: Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111),
04:00:59.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:00:59.345118 2380 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:00:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:59.377238 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:00:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:00:59.377539 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:00:59.580 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:00:59.581 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:00:59.581 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:59.582 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:59.582 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:00:59.583 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:59.583 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:59.583 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:00:59.585 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:01:00.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:01:00.380504 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:00.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:01:00.380789 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:00.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:01:00.771548 2524 raft_consensus.cc:482] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:01:00.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:01:00.771962 2524 raft_consensus.cc:504] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "afd283f610b941d280ee0e72ba536b9c" member_type: VOTER last_known_addr { host: "127.1.229.62" port: 41303 } } peers { permanent_uuid: "3a1de6b27361498c8561cb68eb89897c" member_type: VOTER last_known_addr { host: "127.1.229.61" port: 46763 } } peers { permanent_uuid: "25d212d26cb54c6db7ea1fe7eb95494b" member_type: VOTER last_known_addr { host: "127.1.229.60" port: 43243 } }
04:01:00.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:01:00.773707 2524 leader_election.cc:288] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303), 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763)
04:01:00.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:01:00.776257 2401 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111) [suppressed 1 similar messages]
04:01:00.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:01:00.779218 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer afd283f610b941d280ee0e72ba536b9c (127.1.229.62:41303): Network error: Client connection negotiation failed: client connection to 127.1.229.62:41303: connect: Connection refused (error 111)
04:01:00.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:01:00.779639 2401 leader_election.cc:334] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3a1de6b27361498c8561cb68eb89897c (127.1.229.61:46763): Network error: Client connection negotiation failed: client connection to 127.1.229.61:46763: connect: Connection refused (error 111)
04:01:00.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:01:00.779881 2401 leader_election.cc:302] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 25d212d26cb54c6db7ea1fe7eb95494b; no voters: 3a1de6b27361498c8561cb68eb89897c, afd283f610b941d280ee0e72ba536b9c
04:01:00.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:01:00.780381 2524 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:01:01.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:01:01.383631 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:01.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:01:01.383944 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:02.340 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:01:02.341 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:01:02.341 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:01:02.341 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:01:02.341 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:01:02.342 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:01:02.342 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:01:02.342 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:01:02.345 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:01:02.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:01:02.386951 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:02.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:01:02.387267 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:03.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:01:03.390803 2464 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 25d212d26cb54c6db7ea1fe7eb95494b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:01:03.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 04:01:03.391140 2464 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:01:04.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 04:01:04 dist-test-slave-972x krb5kdc[1952](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781841664, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.229.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:01:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:01:04.015064 2393 init.cc:295] Successfully reacquired a new kerberos TGT
04:01:04.120 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.62:41303(127.1.229.62:41303)] connecting to peer
04:01:04.121 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.229.61:46763(127.1.229.61:46763)] connecting to peer
04:01:04.121 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:01:04.122 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.229.62:41303(127.1.229.62:41303)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:01:04.122 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.62:41303: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303
04:01:04.123 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:01:04.123 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.61:46763(127.1.229.61:46763)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:01:04.123 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.229.61:46763: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
04:01:04.125 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243; will retry
04:01:04.130 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29967), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [185ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [558ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [562ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [577ms] refreshing cache from master, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [584ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [585ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [591ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [593ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [596ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [598ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [622ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [644ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [647ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [692ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [693ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [695ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [723ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [727ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [728ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [802ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [803ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [897ms] refreshing cache from master, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [903ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [905ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [906ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [982ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [982ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [983ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [984ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [990ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [991ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1216ms] refreshing cache from master, [1217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [1220ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [1223ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1224ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [1225ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1236ms] refreshing cache from master, [1237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [1239ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [1241ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [1243ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2076ms] refreshing cache from master, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [2078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [2079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [2079ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [2082ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2084ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [2084ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2916ms] refreshing cache from master, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [2918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [2921ms] too many traces: truncated at 100 traces, deferred=Deferred@1319255028(state=PENDING, result=null, callback=(continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851) -> (continuation of Deferred@595380311 after retry RPC after error@232123233) -> (continuation of Deferred@482284684 after retry RPC after error@1531356460) -> (continuation of Deferred@515764323 after retry RPC after error@1007787574) -> (continuation of Deferred@1483667020 after retry RPC after error@921655208) -> (continuation of Deferred@1193318398 after retry RPC after error@2003849474) -> (continuation of Deferred@1484981558 after retry RPC after error@337468313) -> (continuation of Deferred@1472307587 after retry RPC after error@575410744), errback=(continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851) -> (continuation of Deferred@595380311 after retry RPC after error@232123233) -> (continuation of Deferred@482284684 after retry RPC after error@1531356460) -> (continuation of Deferred@515764323 after retry RPC after error@1007787574) -> (continuation of Deferred@1483667020 after retry RPC after error@921655208) -> (continuation of Deferred@1193318398 after retry RPC after error@2003849474) -> (continuation of Deferred@1484981558 after retry RPC after error@337468313) -> (continuation of Deferred@1472307587 after retry RPC after error@575410744)))
04:01:04.136 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.229.60:43243(127.1.229.60:43243)] disconnecting while in state READY
04:01:04.137 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.229.62:41303(127.1.229.62:41303)] disconnecting while in state TERMINATED
04:01:04.137 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.229.61:46763(127.1.229.61:46763)] disconnecting while in state TERMINATED
04:01:04.142 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
04:01:04.146 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
04:01:04.146 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.229.60:43243(127.1.229.60:43243)] handling channelInactive
04:01:04.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.229.60:43243(127.1.229.60:43243)] cleaning up while in state READY due to: connection closed
04:01:04.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:01:04.151595 1940 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu with pid 2380
04:01:05.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T04:01:05Z chronyd exiting
04:01:06.418 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
04:01:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:01:06.419817 1207 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu with pid 1487
04:01:06.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:01:06.444723 1207 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu with pid 1654
04:01:06.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:01:06.474368 1207 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu with pid 1795
04:01:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:01:06.501612 1207 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu with pid 1219
04:01:06.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:01:06.533077 1207 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu with pid 1289
04:01:06.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 04:01:06.562131 1207 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJlmI1R/build/tsan/bin/kudu with pid 1364
04:01:07.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T04:01:07Z chronyd exiting
04:01:08.753 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29967), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [185ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [558ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [562ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [577ms] refreshing cache from master, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [584ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [585ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [591ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [593ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [596ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [598ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [622ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [644ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [647ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [692ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [693ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [695ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [723ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [727ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [728ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [802ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [803ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [897ms] refreshing cache from master, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [903ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [905ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [906ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [982ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [982ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [983ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [984ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [990ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [991ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1216ms] refreshing cache from master, [1217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [1220ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [1223ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1224ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [1225ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1236ms] refreshing cache from master, [1237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [1239ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [1241ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [1243ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2076ms] refreshing cache from master, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [2078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [2079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [2079ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [2082ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2084ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [2084ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2916ms] refreshing cache from master, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [2918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [2921ms] too many traces: truncated at 100 traces, deferred=Deferred@1319255028(state=PENDING, result=null, callback=(continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851) -> (continuation of Deferred@595380311 after retry RPC after error@232123233) -> (continuation of Deferred@482284684 after retry RPC after error@1531356460) -> (continuation of Deferred@515764323 after retry RPC after error@1007787574) -> (continuation of Deferred@1483667020 after retry RPC after error@921655208) -> (continuation of Deferred@1193318398 after retry RPC after error@2003849474) -> (continuation of Deferred@1484981558 after retry RPC after error@337468313) -> (continuation of Deferred@1472307587 after retry RPC after error@575410744), errback=(continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851) -> (continuation of Deferred@595380311 after retry RPC after error@232123233) -> (continuation of Deferred@482284684 after retry RPC after error@1531356460) -> (continuation of Deferred@515764323 after retry RPC after error@1007787574) -> (continuation of Deferred@1483667020 after retry RPC after error@921655208) -> (continuation of Deferred@1193318398 after retry RPC after error@2003849474) -> (continuation of Deferred@1484981558 after retry RPC after error@337468313) -> (continuation of Deferred@1472307587 after retry RPC after error@575410744)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.229.61:46763
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29967), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [185ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [558ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [562ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [577ms] refreshing cache from master, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [584ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [585ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [591ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [593ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [596ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [598ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [622ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [644ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [647ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [692ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [693ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [695ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [723ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [727ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [728ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [802ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [803ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [897ms] refreshing cache from master, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [903ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [905ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [906ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [982ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [982ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [983ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [984ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [990ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [991ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1216ms] refreshing cache from master, [1217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [1220ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [1223ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1224ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [1225ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1236ms] refreshing cache from master, [1237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [1239ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [1241ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [1243ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2076ms] refreshing cache from master, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [2078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [2079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [2079ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [2082ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2084ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [2084ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2916ms] refreshing cache from master, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [2918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [2921ms] too many traces: truncated at 100 traces, deferred=Deferred@1319255028(state=PENDING, result=null, callback=(continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851) -> (continuation of Deferred@595380311 after retry RPC after error@232123233) -> (continuation of Deferred@482284684 after retry RPC after error@1531356460) -> (continuation of Deferred@515764323 after retry RPC after error@1007787574) -> (continuation of Deferred@1483667020 after retry RPC after error@921655208) -> (continuation of Deferred@1193318398 after retry RPC after error@2003849474) -> (continuation of Deferred@1484981558 after retry RPC after error@337468313) -> (continuation of Deferred@1472307587 after retry RPC after error@575410744), errback=(continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851) -> (continuation of Deferred@595380311 after retry RPC after error@232123233) -> (continuation of Deferred@482284684 after retry RPC after error@1531356460) -> (continuation of Deferred@515764323 after retry RPC after error@1007787574) -> (continuation of Deferred@1483667020 after retry RPC after error@921655208) -> (continuation of Deferred@1193318398 after retry RPC after error@2003849474) -> (continuation of Deferred@1484981558 after retry RPC after error@337468313) -> (continuation of Deferred@1472307587 after retry RPC after error@575410744)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.229.61:46763
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
04:01:09.147 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
04:01:09.147 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29967), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [185ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [558ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [562ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [577ms] refreshing cache from master, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [584ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [585ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [591ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [593ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [596ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [598ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [622ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [644ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [647ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [692ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [693ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [695ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [723ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [727ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [728ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [802ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [803ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [897ms] refreshing cache from master, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [903ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [905ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [906ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [982ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [982ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [983ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [984ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [990ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [991ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1216ms] refreshing cache from master, [1217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [1220ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [1223ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1224ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [1225ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1236ms] refreshing cache from master, [1237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [1239ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [1241ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [1243ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2076ms] refreshing cache from master, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [2078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [2079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [2079ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [2082ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2084ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [2084ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2916ms] refreshing cache from master, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [2918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [2921ms] too many traces: truncated at 100 traces, deferred=Deferred@1319255028(state=PENDING, result=null, callback=(continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851) -> (continuation of Deferred@595380311 after retry RPC after error@232123233) -> (continuation of Deferred@482284684 after retry RPC after error@1531356460) -> (continuation of Deferred@515764323 after retry RPC after error@1007787574) -> (continuation of Deferred@1483667020 after retry RPC after error@921655208) -> (continuation of Deferred@1193318398 after retry RPC after error@2003849474) -> (continuation of Deferred@1484981558 after retry RPC after error@337468313) -> (continuation of Deferred@1472307587 after retry RPC after error@575410744), errback=(continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851) -> (continuation of Deferred@595380311 after retry RPC after error@232123233) -> (continuation of Deferred@482284684 after retry RPC after error@1531356460) -> (continuation of Deferred@515764323 after retry RPC after error@1007787574) -> (continuation of Deferred@1483667020 after retry RPC after error@921655208) -> (continuation of Deferred@1193318398 after retry RPC after error@2003849474) -> (continuation of Deferred@1484981558 after retry RPC after error@337468313) -> (continuation of Deferred@1472307587 after retry RPC after error@575410744)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.229.61:46763
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=26, TimeoutTracker(timeout=30000, elapsed=29967), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [185ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [558ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [562ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [577ms] refreshing cache from master, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [584ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [585ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [591ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [593ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [596ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [598ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [622ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [622ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [626ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [644ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [647ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [682ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [692ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [693ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [695ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [723ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [727ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [728ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [802ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [803ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [897ms] refreshing cache from master, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [903ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [905ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [906ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [982ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [982ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [983ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [984ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [990ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [991ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1216ms] refreshing cache from master, [1217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [1220ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [1223ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1224ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [1225ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1236ms] refreshing cache from master, [1237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [1239ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [1241ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [1243ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2076ms] refreshing cache from master, [2077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [2078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [2079ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [2079ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.62:41303: Network error: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303, [2082ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.61:46763: Network error: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2084ms] Sub RPC ConnectToMaster: received response from server master-127.1.229.60:43243: OK, [2084ms] delaying RPC due to: Service unavailable: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763, [2916ms] refreshing cache from master, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.62:41303, [2917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.61:46763, [2918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.229.60:43243, [2921ms] too many traces: truncated at 100 traces, deferred=Deferred@1319255028(state=PENDING, result=null, callback=(continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851) -> (continuation of Deferred@595380311 after retry RPC after error@232123233) -> (continuation of Deferred@482284684 after retry RPC after error@1531356460) -> (continuation of Deferred@515764323 after retry RPC after error@1007787574) -> (continuation of Deferred@1483667020 after retry RPC after error@921655208) -> (continuation of Deferred@1193318398 after retry RPC after error@2003849474) -> (continuation of Deferred@1484981558 after retry RPC after error@337468313) -> (continuation of Deferred@1472307587 after retry RPC after error@575410744), errback=(continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478) -> (continuation of Deferred@1350021425 after retry RPC after error@759097035) -> (continuation of Deferred@1296026976 after retry RPC after error@1775925605) -> (continuation of Deferred@80327510 after retry RPC after error@1578833338) -> (continuation of Deferred@1836877780 after retry RPC after error@1473759823) -> (continuation of Deferred@508395744 after retry RPC after error@410906710) -> (continuation of Deferred@297174698 after retry RPC after error@2019210081) -> (continuation of Deferred@174224906 after retry RPC after error@2112538704) -> (continuation of Deferred@1489801873 after retry RPC after error@1973482322) -> (continuation of Deferred@1818954186 after retry RPC after error@1206445151) -> (continuation of Deferred@2064428585 after retry RPC after error@155510905) -> (continuation of Deferred@1029014978 after retry RPC after error@1033847851) -> (continuation of Deferred@595380311 after retry RPC after error@232123233) -> (continuation of Deferred@482284684 after retry RPC after error@1531356460) -> (continuation of Deferred@515764323 after retry RPC after error@1007787574) -> (continuation of Deferred@1483667020 after retry RPC after error@921655208) -> (continuation of Deferred@1193318398 after retry RPC after error@2003849474) -> (continuation of Deferred@1484981558 after retry RPC after error@337468313) -> (continuation of Deferred@1472307587 after retry RPC after error@575410744)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.229.62:41303,127.1.229.61:46763,127.1.229.60:43243) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.62:41303(127.1.229.62:41303): Connection refused: /127.1.229.62:41303,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.229.61:46763(127.1.229.61:46763): Connection refused: /127.1.229.61:46763
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.229.61:46763
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)