15:15:07.592 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:15:07.602 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskqZPCv2/test-tmp
15:15:07.605 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:15:07.608 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskqZPCv2/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:15:07.813 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:15:08.013 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:15:08.014 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:15:08.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:08.017346 22808 mini_oidc.cc:152] Starting JWKS server
15:15:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:08.021385 22808 webserver.cc:460] Webserver started at http://127.0.0.1:37115/ using document root <none> and password file <none>
15:15:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:08.021826 22808 mini_oidc.cc:189] Starting OIDC Discovery server
15:15:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:08.023758 22808 webserver.cc:460] Webserver started at http://127.0.0.1:45483/ using document root <none> and password file <none>
15:15:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-23T15:15:08Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
15:15:08.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-23T15:15:08Z Disabled control of system clock
15:15:08.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:08.078992 22808 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:08.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:08.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/wal
15:15:08.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/data
15:15:08.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:15:08.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:15:08.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:15:08.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:08.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:08.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:08.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/logs
15:15:08.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/data/info.pb
15:15:08.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:08.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:15:08.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:15:08.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:15:08.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:15:08.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:08.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:08.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:08.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:15:08.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:15:08.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:08.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:08.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.70.62:46559
15:15:08.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.70.62
15:15:08.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:08.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.70.20:40465
15:15:08.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:08.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:08.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:08.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:08.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.70.62:46559,127.22.70.61:42087,127.22.70.60:37597
15:15:08.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:08.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37115/jwks.json with env {}
15:15:08.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:08.393375 22820 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:15:08.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:08.393890 22820 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:15:08.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:08.394578 22820 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:15:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:08.424317 22820 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:15:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:08.424558 22820 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:15:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:08.424791 22820 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:15:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:08.425094 22820 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:15:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:08.425436 22820 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:15:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:08.425686 22820 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37115/jwks.json
15:15:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:08.458485 22820 master_runner.cc:386] Master server non-default flags:
15:15:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.70.20:40465
15:15:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:08.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/data
15:15:08.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/wal
15:15:08.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:08.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.70.62:46559,127.22.70.61:42087,127.22.70.60:37597
15:15:08.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:08.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:08.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.70.62:46559
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37115/jwks.json
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/data/info.pb
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.70.62
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:15:08.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:15:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/logs
15:15:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:15:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:15:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:15:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:15:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:15:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:15:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 14:32:06 UTC on ccycloud.quasar-akrprb.root.comops.site
15:15:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4667498
15:15:08.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:15:08.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:08.460139 22820 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:15:08.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:08.462168 22820 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:15:08.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:08.482584 22827 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:09.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:08.482844 22829 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:09.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:08.482856 22826 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:09.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:08.484022 22820 server_base.cc:1034] running on GCE node
15:15:09.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.663452 22820 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:15:09.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.666003 22820 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:15:09.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.667294 22820 hybrid_clock.cc:648] HybridClock initialized: now 1790176509667232 us; error 108 us; skew 500 ppm
15:15:09.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.668047 22820 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:15:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.680291 22820 webserver.cc:460] Webserver started at http://127.22.70.62:39003/ using document root <none> and password file <none>
15:15:09.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.681142 22820 fs_manager.cc:359] Metadata directory not provided
15:15:09.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.681313 22820 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:15:09.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.681660 22820 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:15:09.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.686228 22820 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/data/instance:
15:15:09.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdfaa1e5890e44fda2de81745f95c468"
15:15:09.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:09 on dist-test-slave-7664"
15:15:09.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.687369 22820 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/wal/instance:
15:15:09.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdfaa1e5890e44fda2de81745f95c468"
15:15:09.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:09 on dist-test-slave-7664"
15:15:09.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.695837 22820 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
15:15:09.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.701367 22836 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:15:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.702425 22820 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
15:15:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.702735 22820 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/data,/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/wal
15:15:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bdfaa1e5890e44fda2de81745f95c468"
15:15:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:09 on dist-test-slave-7664"
15:15:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.703015 22820 fs_report.cc:383] FS layout report
15:15:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:15:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/wal
15:15:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/wal
15:15:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/data/data
15:15:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:15:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:15:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:15:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:15:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:15:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:15:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:15:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:15:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:15:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:15:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:15:09.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.736011 22820 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:15:09.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.737265 22820 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:15:09.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.737636 22820 kserver.cc:163] Server-wide thread pool size limit: 3276
15:15:09.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.827651 22820 rpc_server.cc:307] RPC server started. Bound to: 127.22.70.62:46559
15:15:09.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.827728 22887 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.70.62:46559 every 8 connection(s)
15:15:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.832873 22820 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/data/info.pb
15:15:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.838392 22888 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:15:09.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.843299 22808 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu as pid 22820
15:15:09.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.843644 22808 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-0/wal/instance
15:15:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.846688 22808 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/wal
15:15:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/data
15:15:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:15:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:15:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:15:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:09.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/logs
15:15:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/data/info.pb
15:15:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:15:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:15:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:15:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:15:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:15:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:15:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.70.61:42087
15:15:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.70.61
15:15:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.70.20:40465
15:15:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.70.62:46559,127.22.70.61:42087,127.22.70.60:37597
15:15:09.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37115/jwks.json with env {}
15:15:09.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.851733 22888 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } has no permanent_uuid. Determining permanent_uuid...
15:15:09.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.886684 22888 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } has no permanent_uuid. Determining permanent_uuid...
15:15:09.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:09.889182 22839 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.70.61:42087: connect: Connection refused (error 111)
15:15:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:09.896557 22888 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.61:42087: Network error: Client connection negotiation failed: client connection to 127.22.70.61:42087: connect: Connection refused (error 111)
15:15:09.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:09.946069 22888 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } attempt: 1
15:15:09.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:09.949965 22888 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.61:42087: Network error: Client connection negotiation failed: client connection to 127.22.70.61:42087: connect: Connection refused (error 111)
15:15:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:10.018356 22888 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } attempt: 2
15:15:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.022861 22888 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.61:42087: Network error: Client connection negotiation failed: client connection to 127.22.70.61:42087: connect: Connection refused (error 111)
15:15:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:10.114195 22888 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } attempt: 3
15:15:10.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.117769 22888 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.61:42087: Network error: Client connection negotiation failed: client connection to 127.22.70.61:42087: connect: Connection refused (error 111)
15:15:10.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.173877 22890 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:15:10.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.174427 22890 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:15:10.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.175037 22890 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:15:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.203420 22890 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:15:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.203650 22890 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:15:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.203819 22890 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:15:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.203976 22890 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:15:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.204149 22890 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:15:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.204403 22890 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37115/jwks.json
15:15:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:10.236547 22890 master_runner.cc:386] Master server non-default flags:
15:15:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.70.20:40465
15:15:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/data
15:15:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/wal
15:15:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.70.62:46559,127.22.70.61:42087,127.22.70.60:37597
15:15:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.70.61:42087
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37115/jwks.json
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/data/info.pb
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.70.61
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/logs
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:15:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:15:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:15:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:15:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:15:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:15:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 14:32:06 UTC on ccycloud.quasar-akrprb.root.comops.site
15:15:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4667498
15:15:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:15:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:10.237641 22890 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:15:10.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:10.239658 22890 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:15:10.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.258251 22900 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:10.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:10.261215 22888 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } attempt: 4
15:15:10.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.264919 22888 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.61:42087: Network error: Client connection negotiation failed: client connection to 127.22.70.61:42087: connect: Connection refused (error 111)
15:15:10.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:10.564383 22888 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } attempt: 5
15:15:10.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.568853 22888 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.61:42087: Network error: Client connection negotiation failed: client connection to 127.22.70.61:42087: connect: Connection refused (error 111)
15:15:11.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.116233 22888 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } attempt: 6
15:15:11.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.123592 22888 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.61:42087: Network error: Client connection negotiation failed: client connection to 127.22.70.61:42087: connect: Connection refused (error 111)
15:15:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.258345 22899 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:11.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:10.258257 22902 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:11.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.398428 22901 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:15:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.398541 22890 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:15:11.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.402725 22890 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:15:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.406762 22890 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:15:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.408242 22890 hybrid_clock.cc:648] HybridClock initialized: now 1790176511408196 us; error 53 us; skew 500 ppm
15:15:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.409154 22890 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:15:11.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.416705 22890 webserver.cc:460] Webserver started at http://127.22.70.61:34137/ using document root <none> and password file <none>
15:15:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.417691 22890 fs_manager.cc:359] Metadata directory not provided
15:15:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.417912 22890 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:15:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.418550 22890 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:15:11.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.424082 22890 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/data/instance:
15:15:11.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed1ee1ed623c47789458e6ecf38da3fd"
15:15:11.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:11 on dist-test-slave-7664"
15:15:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.425340 22890 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/wal/instance:
15:15:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed1ee1ed623c47789458e6ecf38da3fd"
15:15:11.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:11 on dist-test-slave-7664"
15:15:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.433041 22890 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
15:15:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.438669 22911 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:15:11.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.439661 22890 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
15:15:11.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.440049 22890 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/data,/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/wal
15:15:11.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed1ee1ed623c47789458e6ecf38da3fd"
15:15:11.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:11 on dist-test-slave-7664"
15:15:11.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.440444 22890 fs_report.cc:383] FS layout report
15:15:11.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:15:11.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/wal
15:15:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/wal
15:15:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/data/data
15:15:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:15:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:15:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:15:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:15:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:15:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:15:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:15:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:15:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:15:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:15:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:15:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.484345 22890 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:15:11.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.486143 22890 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:15:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.486812 22890 kserver.cc:163] Server-wide thread pool size limit: 3276
15:15:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.572708 22890 rpc_server.cc:307] RPC server started. Bound to: 127.22.70.61:42087
15:15:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.572775 22962 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.70.61:42087 every 8 connection(s)
15:15:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.575677 22890 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/data/info.pb
15:15:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.581207 22963 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:15:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.581337 22808 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu as pid 22890
15:15:11.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.581842 22808 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-1/wal/instance
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.584919 22808 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/wal
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/data
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/logs
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/data/info.pb
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:15:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.70.60:37597
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.70.60
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.70.20:40465
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.70.62:46559,127.22.70.61:42087,127.22.70.60:37597
15:15:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37115/jwks.json with env {}
15:15:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.596630 22963 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } has no permanent_uuid. Determining permanent_uuid...
15:15:11.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.624204 22963 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } has no permanent_uuid. Determining permanent_uuid...
15:15:11.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.639350 22963 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } has no permanent_uuid. Determining permanent_uuid...
15:15:11.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.641701 22912 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:11.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.645833 22963 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.60:37597: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:11.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.695317 22963 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } attempt: 1
15:15:11.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.699115 22963 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.60:37597: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.767498 22963 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } attempt: 2
15:15:11.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.771032 22963 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.60:37597: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.862375 22963 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } attempt: 3
15:15:11.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.866154 22963 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.60:37597: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:11.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.925153 22965 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:15:11.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.925823 22965 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:15:11.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.926553 22965 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:15:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.955518 22965 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:15:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.955863 22965 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:15:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.956261 22965 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:15:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.956609 22965 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:15:11.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.956948 22965 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:15:11.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:11.957188 22965 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37115/jwks.json
15:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.989655 22965 master_runner.cc:386] Master server non-default flags:
15:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.70.20:40465
15:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/data
15:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/wal
15:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.70.62:46559,127.22.70.61:42087,127.22.70.60:37597
15:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.70.60:37597
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37115/jwks.json
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/data/info.pb
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.70.60
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/logs
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:15:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:15:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:15:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:15:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:15:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 14:32:06 UTC on ccycloud.quasar-akrprb.root.comops.site
15:15:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4667498
15:15:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:15:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.991211 22965 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:15:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:11.993175 22965 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:15:12.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:12.009626 22963 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } attempt: 4
15:15:12.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:12.014487 22963 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.60:37597: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:12.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:12.014525 22976 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:12.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:12.183984 22888 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } attempt: 7
15:15:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:12.218109 22888 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } has no permanent_uuid. Determining permanent_uuid...
15:15:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:12.225536 22888 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.60:37597: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:12.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:12.284044 22888 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } attempt: 1
15:15:12.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:12.289121 22888 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.60:37597: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:12.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:12.313912 22963 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } attempt: 5
15:15:12.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:12.319401 22963 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.60:37597: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:12.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:12.370625 22888 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } attempt: 2
15:15:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:12.375892 22888 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.60:37597: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:12.461378 22888 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } attempt: 3
15:15:12.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:12.467110 22888 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.60:37597: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:12.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:12.607632 22888 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } attempt: 4
15:15:12.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:12.615681 22888 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.60:37597: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:12.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:12.866896 22963 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } attempt: 6
15:15:12.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:12.877296 22963 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.60:37597: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:12.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:12.899251 22888 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } attempt: 5
15:15:12.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:12.904672 22888 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.70.60:37597: Network error: Client connection negotiation failed: client connection to 127.22.70.60:37597: connect: Connection refused (error 111)
15:15:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:12.015017 22978 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:12.019568 22975 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:13.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:13.212281 22977 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1195 milliseconds
15:15:13.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.212389 22965 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:15:13.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.213554 22965 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:15:13.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.216610 22965 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:15:13.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.218040 22965 hybrid_clock.cc:648] HybridClock initialized: now 1790176513218017 us; error 45 us; skew 500 ppm
15:15:13.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.219111 22965 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:15:13.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.226569 22965 webserver.cc:460] Webserver started at http://127.22.70.60:46815/ using document root <none> and password file <none>
15:15:13.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.227710 22965 fs_manager.cc:359] Metadata directory not provided
15:15:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.227970 22965 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:15:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.228623 22965 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:15:13.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.236111 22965 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/data/instance:
15:15:13.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb4c9297ea5d42dd9b4393c4949f8df2"
15:15:13.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:13 on dist-test-slave-7664"
15:15:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.237653 22965 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/wal/instance:
15:15:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb4c9297ea5d42dd9b4393c4949f8df2"
15:15:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:13 on dist-test-slave-7664"
15:15:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.248639 22965 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.007s sys 0.005s
15:15:13.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.255581 22990 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:15:13.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.256623 22965 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.256904 22965 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/data,/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/wal
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb4c9297ea5d42dd9b4393c4949f8df2"
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:13 on dist-test-slave-7664"
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.257193 22965 fs_report.cc:383] FS layout report
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/wal
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/wal
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/data/data
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:15:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:15:13.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:15:13.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:15:13.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:15:13.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:15:13.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.294880 22965 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:15:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.296281 22965 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:15:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.296636 22965 kserver.cc:163] Server-wide thread pool size limit: 3276
15:15:13.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.370039 22965 rpc_server.cc:307] RPC server started. Bound to: 127.22.70.60:37597
15:15:13.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.370121 23041 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.70.60:37597 every 8 connection(s)
15:15:13.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.372892 22965 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/data/info.pb
15:15:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.376905 22808 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu as pid 22965
15:15:13.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.377427 22808 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/master-2/wal/instance
15:15:13.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.380649 23042 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:15:13.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.401211 23042 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } has no permanent_uuid. Determining permanent_uuid...
15:15:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.425633 23042 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } has no permanent_uuid. Determining permanent_uuid...
15:15:13.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.437438 23042 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } has no permanent_uuid. Determining permanent_uuid...
15:15:13.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.453553 23042 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2: Bootstrap starting.
15:15:13.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.457216 22888 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } attempt: 6
15:15:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.462445 23042 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2: Neither blocks nor log segments found. Creating new log.
15:15:13.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.464826 23042 log.cc:824] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2: Log is configured to *not* fsync() on all Append() calls
15:15:13.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.471923 23042 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2: No bootstrap required, opened a new log
15:15:13.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.479888 22888 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468: Bootstrap starting.
15:15:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.485639 22888 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468: Neither blocks nor log segments found. Creating new log.
15:15:13.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.487550 22888 log.cc:824] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468: Log is configured to *not* fsync() on all Append() calls
15:15:13.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.493116 22888 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468: No bootstrap required, opened a new log
15:15:13.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.496913 23042 raft_consensus.cc:348] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } }
15:15:13.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.497444 23042 raft_consensus.cc:374] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:15:13.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.497656 23042 raft_consensus.cc:729] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eb4c9297ea5d42dd9b4393c4949f8df2, State: Initialized, Role: FOLLOWER
15:15:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.498541 23042 consensus_queue.cc:260] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } }
15:15:13.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.501770 23053 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } } }
15:15:13.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.502890 23053 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [sys.catalog]: This master's current role is: FOLLOWER
15:15:13.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.503824 23042 sys_catalog.cc:564] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [sys.catalog]: configured and running, proceeding with master startup.
15:15:13.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.526662 22888 raft_consensus.cc:348] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } }
15:15:13.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.527591 22888 raft_consensus.cc:374] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:15:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.527889 22888 raft_consensus.cc:729] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bdfaa1e5890e44fda2de81745f95c468, State: Initialized, Role: FOLLOWER
15:15:13.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.528812 22888 consensus_queue.cc:260] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } }
15:15:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.531857 23062 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } } }
15:15:13.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.532894 23062 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [sys.catalog]: This master's current role is: FOLLOWER
15:15:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.533736 22888 sys_catalog.cc:564] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [sys.catalog]: configured and running, proceeding with master startup.
15:15:13.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:13.551991 22996 tablet.cc:2369] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:15:13.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:13.553689 23065 catalog_manager.cc:1558] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:13.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:13.554160 23065 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:13.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.544240 22965 master_runner.cc:186] Error getting master registration for 127.22.70.62:46559: OK, instance_id { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" instance_seqno: 1790176509802945 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:15:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.581907 22965 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:15:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:13.598325 23078 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:13.598634 23078 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:13.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.584364 22820 master_runner.cc:186] Error getting master registration for 127.22.70.61:42087: OK, instance_id { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" instance_seqno: 1790176511547965 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:15:13.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.621058 22820 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:15:13.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:13.747022 22842 tablet.cc:2369] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:15:13.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.824494 23053 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:15:13.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.825047 23053 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } }
15:15:13.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.827450 23053 leader_election.cc:288] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bdfaa1e5890e44fda2de81745f95c468 (127.22.70.62:46559), ed1ee1ed623c47789458e6ecf38da3fd (127.22.70.61:42087)
15:15:13.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.830655 22863 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bdfaa1e5890e44fda2de81745f95c468" is_pre_election: true
15:15:13.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.831756 22863 raft_consensus.cc:2454] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eb4c9297ea5d42dd9b4393c4949f8df2 in term 0.
15:15:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.833082 22993 leader_election.cc:302] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: bdfaa1e5890e44fda2de81745f95c468, eb4c9297ea5d42dd9b4393c4949f8df2; no voters:
15:15:13.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.834403 23053 raft_consensus.cc:2781] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:15:13.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.835253 23053 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:15:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.835969 23053 raft_consensus.cc:3037] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [term 0 FOLLOWER]: Advancing to term 1
15:15:13.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.850103 23053 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } }
15:15:13.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.853701 22863 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bdfaa1e5890e44fda2de81745f95c468"
15:15:13.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.854197 22863 raft_consensus.cc:3037] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [term 0 FOLLOWER]: Advancing to term 1
15:15:13.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.866381 22863 raft_consensus.cc:2454] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eb4c9297ea5d42dd9b4393c4949f8df2 in term 1.
15:15:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.872506 22993 leader_election.cc:302] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: bdfaa1e5890e44fda2de81745f95c468, eb4c9297ea5d42dd9b4393c4949f8df2; no voters:
15:15:13.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.878690 23053 leader_election.cc:288] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [CANDIDATE]: Term 1 election: Requested vote from peers bdfaa1e5890e44fda2de81745f95c468 (127.22.70.62:46559), ed1ee1ed623c47789458e6ecf38da3fd (127.22.70.61:42087)
15:15:13.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.879089 23053 raft_consensus.cc:2781] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [term 1 FOLLOWER]: Leader election won for term 1
15:15:13.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.879459 23053 raft_consensus.cc:686] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [term 1 LEADER]: Becoming Leader. State: Replica: eb4c9297ea5d42dd9b4393c4949f8df2, State: Running, Role: LEADER
15:15:13.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.880249 23053 consensus_queue.cc:237] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } }
15:15:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.902819 23080 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader eb4c9297ea5d42dd9b4393c4949f8df2. Latest consensus state: current_term: 1 leader_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } } }
15:15:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.903792 23080 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [sys.catalog]: This master's current role is: LEADER
15:15:13.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.908305 23082 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
15:15:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.916414 23082 catalog_manager.cc:1484] Initializing Kudu cluster ID...
15:15:13.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.934841 22863 raft_consensus.cc:1261] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [term 1 FOLLOWER]: Refusing update from remote peer eb4c9297ea5d42dd9b4393c4949f8df2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:15:13.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.937737 22963 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } attempt: 7
15:15:13.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.941811 23053 consensus_queue.cc:1035] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:15:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.986227 22963 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd: Bootstrap starting.
15:15:13.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.987236 23062 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [sys.catalog]: SysCatalogTable state changed. Reason: New leader eb4c9297ea5d42dd9b4393c4949f8df2. Latest consensus state: current_term: 1 leader_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } } }
15:15:13.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.988711 23062 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [sys.catalog]: This master's current role is: FOLLOWER
15:15:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:13.995111 22963 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd: Neither blocks nor log segments found. Creating new log.
15:15:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.000128 23083 mvcc.cc:205] Tried to move back new op lower bound from 7332563001053253632 to 7332563000864768000. Current Snapshot: MvccSnapshot[applied={T|T < 7332563001053253632}]
15:15:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.005553 22963 log.cc:824] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd: Log is configured to *not* fsync() on all Append() calls
15:15:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.005553 23080 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } } }
15:15:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.006317 23080 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [sys.catalog]: This master's current role is: LEADER
15:15:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.009344 23082 catalog_manager.cc:1347] Generated new cluster ID: 9739919957c14f17b2f377ce779167eb
15:15:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.009613 23082 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
15:15:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.017234 23062 sys_catalog.cc:455] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } } }
15:15:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.017984 23062 sys_catalog.cc:458] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468 [sys.catalog]: This master's current role is: FOLLOWER
15:15:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.022564 22963 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd: No bootstrap required, opened a new log
15:15:14.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.066465 23082 catalog_manager.cc:1370] Generated new certificate authority record
15:15:14.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.068783 23082 catalog_manager.cc:1504] Loading token signing keys...
15:15:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.088145 22963 raft_consensus.cc:348] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } }
15:15:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.089143 22963 raft_consensus.cc:374] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:15:14.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.089547 22963 raft_consensus.cc:729] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ed1ee1ed623c47789458e6ecf38da3fd, State: Initialized, Role: FOLLOWER
15:15:14.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.092182 22963 consensus_queue.cc:260] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } }
15:15:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.106334 23091 sys_catalog.cc:455] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } } }
15:15:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.112201 22963 sys_catalog.cc:564] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd [sys.catalog]: configured and running, proceeding with master startup.
15:15:14.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.118858 23082 catalog_manager.cc:5868] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2: Generated new TSK 0
15:15:14.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.112498 23091 sys_catalog.cc:458] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd [sys.catalog]: This master's current role is: FOLLOWER
15:15:14.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.112907 22937 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ed1ee1ed623c47789458e6ecf38da3fd"
15:15:14.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.120509 23082 catalog_manager.cc:1514] Initializing in-progress tserver states...
15:15:14.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.113080 22938 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" is_pre_election: true
15:15:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.113621 22936 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd [term 0 FOLLOWER]: Advancing to term 1
15:15:14.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.137365 22936 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd [term 1 FOLLOWER]: Refusing update from remote peer eb4c9297ea5d42dd9b4393c4949f8df2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:15:14.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.137892 22808 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:14.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:14.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/wal
15:15:14.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/data
15:15:14.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:15:14.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:15:14.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:15:14.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:14.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:14.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:14.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/logs
15:15:14.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/data/info.pb
15:15:14.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.70.1:0
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.70.1
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.70.1
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.70.62:46559,127.22.70.61:42087,127.22.70.60:37597
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.70.20:40465
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:15:14.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.141932 23080 consensus_queue.cc:1035] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:15:14.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.195118 23091 sys_catalog.cc:455] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd [sys.catalog]: SysCatalogTable state changed. Reason: New leader eb4c9297ea5d42dd9b4393c4949f8df2. Latest consensus state: current_term: 1 leader_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } } }
15:15:14.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.196600 23091 sys_catalog.cc:458] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd [sys.catalog]: This master's current role is: FOLLOWER
15:15:14.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.196122 23098 sys_catalog.cc:455] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } } }
15:15:14.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.197741 23098 sys_catalog.cc:458] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd [sys.catalog]: This master's current role is: FOLLOWER
15:15:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.239190 23108 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:15:14.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.607712 23078 catalog_manager.cc:1259] Loaded cluster ID: 9739919957c14f17b2f377ce779167eb
15:15:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.619304 23078 catalog_manager.cc:1552] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468: loading cluster ID for follower catalog manager: success
15:15:14.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.642815 23078 catalog_manager.cc:1574] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468: acquiring CA information for follower catalog manager: success
15:15:14.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.648171 23078 catalog_manager.cc:1602] T 00000000000000000000000000000000 P bdfaa1e5890e44fda2de81745f95c468: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:15:14.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:14.719726 23093 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:15:14.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:14.721045 23093 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:15:14.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:14.722124 23093 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:15:14.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.769376 23080 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bdfaa1e5890e44fda2de81745f95c468" member_type: VOTER last_known_addr { host: "127.22.70.62" port: 46559 } } peers { permanent_uuid: "ed1ee1ed623c47789458e6ecf38da3fd" member_type: VOTER last_known_addr { host: "127.22.70.61" port: 42087 } } peers { permanent_uuid: "eb4c9297ea5d42dd9b4393c4949f8df2" member_type: VOTER last_known_addr { host: "127.22.70.60" port: 37597 } } }
15:15:14.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.770110 23080 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb4c9297ea5d42dd9b4393c4949f8df2 [sys.catalog]: This master's current role is: LEADER
15:15:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:14.785490 23093 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:15:14.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:14.787547 23093 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.70.1
15:15:14.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.850533 23093 tablet_server_runner.cc:78] Tablet server non-default flags:
15:15:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.70.20:40465
15:15:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/data
15:15:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/wal
15:15:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.70.1:0
15:15:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:15:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:14.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/data/info.pb
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.70.1
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.70.62:46559,127.22.70.61:42087,127.22.70.60:37597
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.70.1
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/logs
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 14:32:06 UTC on ccycloud.quasar-akrprb.root.comops.site
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4667498
15:15:14.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:15:14.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.852334 23093 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:15:14.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:14.855235 23093 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:15:14.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:14.922382 23118 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:15.275719 23108 catalog_manager.cc:1259] Loaded cluster ID: 9739919957c14f17b2f377ce779167eb
15:15:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:15.276283 23108 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd: loading cluster ID for follower catalog manager: success
15:15:15.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:15.290956 23108 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd: acquiring CA information for follower catalog manager: success
15:15:15.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:15.312392 23108 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ed1ee1ed623c47789458e6ecf38da3fd: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:15:17.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:14.922531 23117 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:17.435608 23093 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.524s user 0.571s sys 1.105s
15:15:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:17.435977 23093 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.524s user 0.571s sys 1.105s
15:15:17.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:16.301990 23116 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 23093
15:15:17.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:17.438046 23120 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:17.444146 23119 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2526 milliseconds
15:15:17.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.444193 23093 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:15:17.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.445708 23093 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:15:17.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.448212 23093 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:15:17.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.449654 23093 hybrid_clock.cc:648] HybridClock initialized: now 1790176517449541 us; error 118 us; skew 500 ppm
15:15:17.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.450528 23093 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:15:17.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.467444 23093 webserver.cc:460] Webserver started at http://127.22.70.1:46423/ using document root <none> and password file <none>
15:15:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.468384 23093 fs_manager.cc:359] Metadata directory not provided
15:15:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.468626 23093 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:15:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.469138 23093 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:15:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.474086 23093 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/data/instance:
15:15:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3707f5373766462da6b94946cb636422"
15:15:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:17 on dist-test-slave-7664"
15:15:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.475554 23093 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/wal/instance:
15:15:17.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3707f5373766462da6b94946cb636422"
15:15:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:17 on dist-test-slave-7664"
15:15:17.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.482932 23093 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
15:15:17.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.489311 23130 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:15:17.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.490577 23093 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
15:15:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.490929 23093 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/data,/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/wal
15:15:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3707f5373766462da6b94946cb636422"
15:15:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:17 on dist-test-slave-7664"
15:15:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.491324 23093 fs_report.cc:383] FS layout report
15:15:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/wal
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/wal
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/data/data
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:15:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:15:17.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.551084 23093 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:15:17.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.553097 23093 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:15:17.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.553642 23093 kserver.cc:163] Server-wide thread pool size limit: 3276
15:15:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.557431 23093 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:15:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.562744 23093 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:15:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.562988 23093 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:15:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.563295 23093 ts_tablet_manager.cc:610] Registered 0 tablets
15:15:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.563500 23093 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:15:17.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.885245 23093 rpc_server.cc:307] RPC server started. Bound to: 127.22.70.1:33437
15:15:17.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.886056 23243 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.70.1:33437 every 8 connection(s)
15:15:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.889880 23093 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/data/info.pb
15:15:17.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.891678 22808 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu as pid 23093
15:15:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.899560 22808 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-0/wal/instance
15:15:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:17.917596 22808 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/wal
15:15:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/data
15:15:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/logs
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/data/info.pb
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:15:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:15:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.70.2:0
15:15:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.70.2
15:15:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.70.2
15:15:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.70.62:46559,127.22.70.61:42087,127.22.70.60:37597
15:15:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.70.20:40465
15:15:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:15:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.001955 23248 heartbeater.cc:347] Connected to a master server at 127.22.70.62:46559
15:15:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.002486 23248 heartbeater.cc:464] Registering TS with master...
15:15:18.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.003593 23248 heartbeater.cc:511] Master 127.22.70.62:46559 requested a full tablet report, sending...
15:15:18.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.006834 22852 ts_manager.cc:194] Registered new tserver with Master: 3707f5373766462da6b94946cb636422 (127.22.70.1:33437)
15:15:18.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.009883 23244 heartbeater.cc:347] Connected to a master server at 127.22.70.60:37597
15:15:18.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.010227 23244 heartbeater.cc:464] Registering TS with master...
15:15:18.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.010435 23245 heartbeater.cc:347] Connected to a master server at 127.22.70.61:42087
15:15:18.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.010768 23245 heartbeater.cc:464] Registering TS with master...
15:15:18.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.011004 23244 heartbeater.cc:511] Master 127.22.70.60:37597 requested a full tablet report, sending...
15:15:18.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.011600 23245 heartbeater.cc:511] Master 127.22.70.61:42087 requested a full tablet report, sending...
15:15:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.013474 23007 ts_manager.cc:194] Registered new tserver with Master: 3707f5373766462da6b94946cb636422 (127.22.70.1:33437)
15:15:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.014392 22928 ts_manager.cc:194] Registered new tserver with Master: 3707f5373766462da6b94946cb636422 (127.22.70.1:33437)
15:15:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.016319 23007 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.22.70.1:45089
15:15:18.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:18.463333 23251 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:15:18.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:18.464250 23251 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:15:18.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:18.465164 23251 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:15:18.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:18.521222 23251 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:15:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:18.522915 23251 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.70.2
15:15:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.580971 23251 tablet_server_runner.cc:78] Tablet server non-default flags:
15:15:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.70.20:40465
15:15:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/data
15:15:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/wal
15:15:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.70.2:0
15:15:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:15:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:18.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:18.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/data/info.pb
15:15:18.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.70.2
15:15:18.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:18.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.70.62:46559,127.22.70.61:42087,127.22.70.60:37597
15:15:18.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.70.2
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/logs
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 14:32:06 UTC on ccycloud.quasar-akrprb.root.comops.site
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4667498
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:15:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.582405 23251 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:15:18.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.584367 23251 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:15:18.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:18.617731 23262 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:19.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.019675 23244 heartbeater.cc:503] Master 127.22.70.60:37597 was elected leader, sending a full tablet report...
15:15:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:18.618265 23263 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:19.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:18.619637 23265 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:19.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:18.619781 23251 server_base.cc:1034] running on GCE node
15:15:19.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.838209 23251 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:15:19.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.841279 23251 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:15:19.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.842747 23251 hybrid_clock.cc:648] HybridClock initialized: now 1790176519842698 us; error 44 us; skew 500 ppm
15:15:19.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.844403 23251 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:15:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.863909 23251 webserver.cc:460] Webserver started at http://127.22.70.2:43789/ using document root <none> and password file <none>
15:15:19.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.864920 23251 fs_manager.cc:359] Metadata directory not provided
15:15:19.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.865093 23251 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:15:19.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.865545 23251 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:15:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.872071 23251 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/data/instance:
15:15:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8785bc5c185e4952b0f50178b8f556fe"
15:15:19.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:19 on dist-test-slave-7664"
15:15:19.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.873095 23251 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/wal/instance:
15:15:19.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8785bc5c185e4952b0f50178b8f556fe"
15:15:19.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:19 on dist-test-slave-7664"
15:15:19.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.885119 23251 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.006s sys 0.005s
15:15:19.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.897202 23272 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:15:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.898844 23251 fs_manager.cc:725] Time spent opening block manager: real 0.009s user 0.005s sys 0.004s
15:15:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.899268 23251 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/data,/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/wal
15:15:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8785bc5c185e4952b0f50178b8f556fe"
15:15:19.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:19 on dist-test-slave-7664"
15:15:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.899699 23251 fs_report.cc:383] FS layout report
15:15:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:15:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/wal
15:15:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/wal
15:15:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/data/data
15:15:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:15:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:15:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:15:19.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:15:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:15:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:15:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:15:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:15:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:15:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:15:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:15:19.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.953902 23251 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:15:19.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.956825 23251 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:15:19.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.957726 23251 kserver.cc:163] Server-wide thread pool size limit: 3276
15:15:19.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.962278 23251 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:15:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.970345 23251 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:15:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.970677 23251 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:15:19.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.971024 23251 ts_tablet_manager.cc:610] Registered 0 tablets
15:15:19.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:19.971288 23251 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.002s sys 0.000s
15:15:20.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.235316 23251 rpc_server.cc:307] RPC server started. Bound to: 127.22.70.2:40257
15:15:20.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.235482 23386 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.70.2:40257 every 8 connection(s)
15:15:20.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.237751 23251 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/data/info.pb
15:15:20.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.239564 22808 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu as pid 23251
15:15:20.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.240232 22808 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-1/wal/instance
15:15:20.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.284839 22808 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:20.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:20.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/wal
15:15:20.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/data
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/logs
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/data/info.pb
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:15:20.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:15:20.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.70.3:0
15:15:20.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.70.3
15:15:20.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.70.3
15:15:20.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:20.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.70.62:46559,127.22.70.61:42087,127.22.70.60:37597
15:15:20.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.70.20:40465
15:15:20.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:20.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:20.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:15:20.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.312633 23387 heartbeater.cc:347] Connected to a master server at 127.22.70.60:37597
15:15:20.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.313133 23387 heartbeater.cc:464] Registering TS with master...
15:15:20.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.314239 23387 heartbeater.cc:511] Master 127.22.70.60:37597 requested a full tablet report, sending...
15:15:20.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.314543 23388 heartbeater.cc:347] Connected to a master server at 127.22.70.61:42087
15:15:20.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.314888 23388 heartbeater.cc:464] Registering TS with master...
15:15:20.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.315748 23388 heartbeater.cc:511] Master 127.22.70.61:42087 requested a full tablet report, sending...
15:15:20.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.316646 23389 heartbeater.cc:347] Connected to a master server at 127.22.70.62:46559
15:15:20.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.316771 23007 ts_manager.cc:194] Registered new tserver with Master: 8785bc5c185e4952b0f50178b8f556fe (127.22.70.2:40257)
15:15:20.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.317112 23389 heartbeater.cc:464] Registering TS with master...
15:15:20.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.318488 23389 heartbeater.cc:511] Master 127.22.70.62:46559 requested a full tablet report, sending...
15:15:20.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.319267 23007 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.22.70.2:37499
15:15:20.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.320209 22928 ts_manager.cc:194] Registered new tserver with Master: 8785bc5c185e4952b0f50178b8f556fe (127.22.70.2:40257)
15:15:20.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.325057 22852 ts_manager.cc:194] Registered new tserver with Master: 8785bc5c185e4952b0f50178b8f556fe (127.22.70.2:40257)
15:15:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:20.745580 23397 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:15:20.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:20.746340 23397 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:15:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:20.746896 23397 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:15:20.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:20.776618 23397 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:15:20.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:20.777297 23397 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.70.3
15:15:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.808049 23397 tablet_server_runner.cc:78] Tablet server non-default flags:
15:15:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.70.20:40465
15:15:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/data
15:15:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/wal
15:15:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.70.3:0
15:15:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:15:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/data/info.pb
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.70.3
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.70.62:46559,127.22.70.61:42087,127.22.70.60:37597
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.70.3
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/logs
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 14:32:06 UTC on ccycloud.quasar-akrprb.root.comops.site
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4667498
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:15:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.809271 23397 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:15:20.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:20.811094 23397 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:15:20.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:20.836254 23403 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:21.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:21.326814 23387 heartbeater.cc:503] Master 127.22.70.60:37597 was elected leader, sending a full tablet report...
15:15:22.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:20.837374 23404 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:20.841493 23406 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:22.111761 23405 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1272 milliseconds
15:15:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.111867 23397 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:15:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.113322 23397 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:15:22.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.115811 23397 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:15:22.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.117172 23397 hybrid_clock.cc:648] HybridClock initialized: now 1790176522117150 us; error 41 us; skew 500 ppm
15:15:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.118046 23397 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:15:22.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.125255 23397 webserver.cc:460] Webserver started at http://127.22.70.3:39635/ using document root <none> and password file <none>
15:15:22.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.126485 23397 fs_manager.cc:359] Metadata directory not provided
15:15:22.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.126734 23397 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:15:22.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.127277 23397 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:15:22.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.132056 23397 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/data/instance:
15:15:22.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bc8c59d646d48e7817a766df3e86408"
15:15:22.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:22 on dist-test-slave-7664"
15:15:22.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.133211 23397 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/wal/instance:
15:15:22.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bc8c59d646d48e7817a766df3e86408"
15:15:22.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:22 on dist-test-slave-7664"
15:15:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.140699 23397 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
15:15:22.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.150017 23416 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:15:22.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.151347 23397 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.003s sys 0.001s
15:15:22.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.151871 23397 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/data,/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/wal
15:15:22.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bc8c59d646d48e7817a766df3e86408"
15:15:22.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:22 on dist-test-slave-7664"
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.152516 23397 fs_report.cc:383] FS layout report
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/wal
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/wal
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/data/data
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:15:22.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:15:22.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.208112 23397 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:15:22.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.209438 23397 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:15:22.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.209829 23397 kserver.cc:163] Server-wide thread pool size limit: 3276
15:15:22.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.212414 23397 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:15:22.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.216265 23397 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:15:22.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.216444 23397 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:15:22.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.216655 23397 ts_tablet_manager.cc:610] Registered 0 tablets
15:15:22.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.216784 23397 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:15:22.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.560590 23397 rpc_server.cc:307] RPC server started. Bound to: 127.22.70.3:33015
15:15:22.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.560700 23529 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.70.3:33015 every 8 connection(s)
15:15:22.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.563381 23397 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/data/info.pb
15:15:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.564709 22808 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu as pid 23397
15:15:22.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.565093 22808 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster117737636628676343/ts-2/wal/instance
15:15:22.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.658301 23530 heartbeater.cc:347] Connected to a master server at 127.22.70.60:37597
15:15:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.658844 23530 heartbeater.cc:464] Registering TS with master...
15:15:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.658964 23532 heartbeater.cc:347] Connected to a master server at 127.22.70.62:46559
15:15:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.659318 23532 heartbeater.cc:464] Registering TS with master...
15:15:22.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.660472 23532 heartbeater.cc:511] Master 127.22.70.62:46559 requested a full tablet report, sending...
15:15:22.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.660951 23530 heartbeater.cc:511] Master 127.22.70.60:37597 requested a full tablet report, sending...
15:15:22.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.663451 22852 ts_manager.cc:194] Registered new tserver with Master: 2bc8c59d646d48e7817a766df3e86408 (127.22.70.3:33015)
15:15:22.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.665194 23007 ts_manager.cc:194] Registered new tserver with Master: 2bc8c59d646d48e7817a766df3e86408 (127.22.70.3:33015)
15:15:22.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.665480 23531 heartbeater.cc:347] Connected to a master server at 127.22.70.61:42087
15:15:22.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.665786 23531 heartbeater.cc:464] Registering TS with master...
15:15:22.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.666499 23531 heartbeater.cc:511] Master 127.22.70.61:42087 requested a full tablet report, sending...
15:15:22.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.667904 23007 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.22.70.3:38975
15:15:22.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.669958 22928 ts_manager.cc:194] Registered new tserver with Master: 2bc8c59d646d48e7817a766df3e86408 (127.22.70.3:33015)
15:15:22.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:22.682297 22808 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
15:15:22.696 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:15:22.697 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:15:22.711 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.70.62"
port: 46559
}
bound_http_address {
host: "127.22.70.62"
port: 39003
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.70.61"
port: 42087
}
bound_http_address {
host: "127.22.70.61"
port: 34137
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.70.60"
port: 37597
}
bound_http_address {
host: "127.22.70.60"
port: 46815
}
}
}
15:15:22.716 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:15:22.718 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.22.70.1"
port: 33437
}
bound_http_address {
host: "127.22.70.1"
port: 46423
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.22.70.2"
port: 40257
}
bound_http_address {
host: "127.22.70.2"
port: 43789
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.22.70.3"
port: 33015
}
bound_http_address {
host: "127.22.70.3"
port: 39635
}
}
}
15:15:22.718 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:15:22.912 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:15:23.030 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskqZPCv2/test-tmp
15:15:23.031 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:15:23.031 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskqZPCv2/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:15:23.034 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
15:15:23.458 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:15:23.459 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:15:23.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:23.461937 23541 mini_oidc.cc:152] Starting JWKS server
15:15:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:23.467685 23541 webserver.cc:460] Webserver started at http://127.0.0.1:35517/ using document root <none> and password file <none>
15:15:23.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:23.468164 23541 mini_oidc.cc:189] Starting OIDC Discovery server
15:15:23.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:23.472216 23541 webserver.cc:460] Webserver started at http://127.0.0.1:34233/ using document root <none> and password file <none>
15:15:23.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
15:15:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/principal' for realm 'KRBTEST.COM',
15:15:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
15:15:23.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:23 dist-test-slave-7664 krb5kdc[23553](info): setting up network...
15:15:23.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
15:15:23.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:23 dist-test-slave-7664 krb5kdc[23553](info): set up 2 sockets
15:15:23.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:23 dist-test-slave-7664 krb5kdc[23553](info): commencing operation
15:15:23.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
15:15:23.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:23.673802 23530 heartbeater.cc:503] Master 127.22.70.60:37597 was elected leader, sending a full tablet report...
15:15:24.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:24.572214 23541 mini_kdc.cc:121] Time spent starting KDC: real 1.067s user 0.003s sys 0.020s
15:15:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
15:15:24.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:15:24.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
15:15:24.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
15:15:24.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:15:24.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
15:15:24.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
15:15:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:15:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
15:15:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:15:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/test-user.keytab.
15:15:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/test-user.keytab.
15:15:24.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:24 dist-test-slave-7664 krb5kdc[23553](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176524, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:15:24.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:15:24.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-23T15:15:24Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
15:15:24.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-23T15:15:24Z Disabled control of system clock
15:15:24.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.253.126@KRBTEST.COM; defaulting to no policy
15:15:24.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:15:24.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.253.126@KRBTEST.COM" created.
15:15:24.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:15:24.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.253.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.126.keytab.
15:15:24.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.253.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.126.keytab.
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:24.777225 23541 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/logs
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data/info.pb
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:24.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.253.126:39397
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.253.126
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.126.keytab
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.253.126
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.253.84:37719
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.253.126:39397,127.22.253.125:41831,127.22.253.124:38783
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35517/jwks.json
15:15:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:15:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.120878 23567 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:15:25.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.121579 23567 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:15:25.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.122498 23567 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:15:25.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.151589 23567 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:15:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.151860 23567 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:15:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.152268 23567 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:15:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.152604 23567 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:15:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.152907 23567 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:15:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.153195 23567 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:15:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.153407 23567 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35517/jwks.json
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:25.187781 23567 master_runner.cc:386] Master server non-default flags:
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.253.84:37719
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.253.126:39397,127.22.253.125:41831,127.22.253.124:38783
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.126.keytab
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.253.126
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.253.126:39397
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35517/jwks.json
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:15:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data/info.pb
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.253.126
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/logs
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 14:32:06 UTC on ccycloud.quasar-akrprb.root.comops.site
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4667498
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:15:25.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:25.189175 23567 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:15:25.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:25.190301 23567 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:15:25.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.216542 23573 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:25.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.595702 23526 debug-util.cc:398] Leaking SignalData structure 0x7b08000af6c0 after lost signal to thread 23398
15:15:25.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.597684 23526 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba340 after lost signal to thread 23529
15:15:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.216676 23574 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:26.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:25.218779 23567 server_base.cc:1034] running on GCE node
15:15:26.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:25.216562 23576 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.475425 23567 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:15:26.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.478506 23567 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:15:26.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.479880 23567 hybrid_clock.cc:648] HybridClock initialized: now 1790176526479859 us; error 40 us; skew 500 ppm
15:15:26.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.480607 23567 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:15:26.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:26 dist-test-slave-7664 krb5kdc[23553](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176526, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:15:26.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.486367 23567 init.cc:377] Logged in from keytab as kudu/127.22.253.126@KRBTEST.COM (short username kudu)
15:15:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.491953 23567 webserver.cc:460] Webserver started at http://127.22.253.126:33229/ using document root <none> and password file <none>
15:15:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.492740 23567 fs_manager.cc:359] Metadata directory not provided
15:15:26.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.492897 23567 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:15:26.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.493247 23567 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:15:26.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.497735 23567 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data/instance:
15:15:26.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abb2d0917adf4fea983f3c1078c99841"
15:15:26.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:26 on dist-test-slave-7664"
15:15:26.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.498673 23567 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal/instance:
15:15:26.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abb2d0917adf4fea983f3c1078c99841"
15:15:26.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:26 on dist-test-slave-7664"
15:15:26.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.506176 23567 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.001s
15:15:26.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.511863 23585 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:15:26.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.513098 23567 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:15:26.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.513417 23567 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data,/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal
15:15:26.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abb2d0917adf4fea983f3c1078c99841"
15:15:26.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:26 on dist-test-slave-7664"
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.513728 23567 fs_report.cc:383] FS layout report
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data/data
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:15:26.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:15:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.565688 23567 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:15:26.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.571578 23567 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:15:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.572337 23567 kserver.cc:163] Server-wide thread pool size limit: 3276
15:15:26.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.692698 23567 rpc_server.cc:307] RPC server started. Bound to: 127.22.253.126:39397
15:15:26.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.692816 23636 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.253.126:39397 every 8 connection(s)
15:15:26.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.695309 23567 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data/info.pb
15:15:26.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.701614 23637 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:15:26.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.702967 23541 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu as pid 23567
15:15:26.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.703572 23541 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal/instance
15:15:26.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.253.125@KRBTEST.COM; defaulting to no policy
15:15:26.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.722820 23637 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } has no permanent_uuid. Determining permanent_uuid...
15:15:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:15:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.253.125@KRBTEST.COM" created.
15:15:26.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:15:26.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.253.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.125.keytab.
15:15:26.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.253.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.125.keytab.
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.757135 23541 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/logs
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data/info.pb
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.253.125:41831
15:15:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.253.125
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.125.keytab
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.253.125
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.253.84:37719
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.253.126:39397,127.22.253.125:41831,127.22.253.124:38783
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35517/jwks.json
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:15:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:26 dist-test-slave-7664 krb5kdc[23553](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176526, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.126@KRBTEST.COM for kudu/127.22.253.126@KRBTEST.COM
15:15:26.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.778942 23637 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } has no permanent_uuid. Determining permanent_uuid...
15:15:26.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:26.781733 23586 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:26.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:26.785866 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.125:41831: Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:26.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.835433 23637 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } attempt: 1
15:15:26.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:26.840039 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.125:41831: Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:26.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:26.908484 23637 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } attempt: 2
15:15:26.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:26.912752 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.125:41831: Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:27.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:27.004177 23637 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } attempt: 3
15:15:27.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.008536 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.125:41831: Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:27.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.087842 23643 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:15:27.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.088539 23643 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:15:27.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.089223 23643 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:15:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.127672 23643 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:15:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.127928 23643 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:15:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.128329 23643 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:15:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.128654 23643 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:15:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.129035 23643 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:15:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.129365 23643 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:15:27.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.129597 23643 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35517/jwks.json
15:15:27.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:27.151935 23637 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } attempt: 4
15:15:27.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.155910 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.125:41831: Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:27.169032 23643 master_runner.cc:386] Master server non-default flags:
15:15:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.253.84:37719
15:15:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data
15:15:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal
15:15:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:15:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.253.126:39397,127.22.253.125:41831,127.22.253.124:38783
15:15:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.125.keytab
15:15:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.253.125
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.253.125:41831
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35517/jwks.json
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data/info.pb
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.253.125
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/logs
15:15:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:15:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:15:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:15:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:15:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:15:27.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:15:27.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 14:32:06 UTC on ccycloud.quasar-akrprb.root.comops.site
15:15:27.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4667498
15:15:27.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:15:27.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:27.170500 23643 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:15:27.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:27.171896 23643 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:15:27.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.202278 23651 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:27.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:27.455366 23637 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } attempt: 5
15:15:27.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.461746 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.125:41831: Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:28.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.009260 23637 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } attempt: 6
15:15:28.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:28.019119 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.125:41831: Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:28.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.209089 23650 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:28.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:27.202709 23653 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:28.524318 23652 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1317 milliseconds
15:15:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.524458 23643 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:15:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.525746 23643 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:15:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.528228 23643 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:15:28.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.529547 23643 hybrid_clock.cc:648] HybridClock initialized: now 1790176528529542 us; error 75 us; skew 500 ppm
15:15:28.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.530308 23643 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:15:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:28 dist-test-slave-7664 krb5kdc[23553](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176528, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:15:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.540191 23643 init.cc:377] Logged in from keytab as kudu/127.22.253.125@KRBTEST.COM (short username kudu)
15:15:28.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.544925 23643 webserver.cc:460] Webserver started at http://127.22.253.125:32975/ using document root <none> and password file <none>
15:15:28.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.545713 23643 fs_manager.cc:359] Metadata directory not provided
15:15:28.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.545879 23643 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:15:28.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.546268 23643 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:15:28.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.550675 23643 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data/instance:
15:15:28.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa26783c67ba45738bae5b848cf9aa9f"
15:15:28.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:28 on dist-test-slave-7664"
15:15:28.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.551802 23643 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal/instance:
15:15:28.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa26783c67ba45738bae5b848cf9aa9f"
15:15:28.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:28 on dist-test-slave-7664"
15:15:28.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.559080 23643 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
15:15:28.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.564070 23666 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:15:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.564992 23643 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
15:15:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.565248 23643 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data,/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal
15:15:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa26783c67ba45738bae5b848cf9aa9f"
15:15:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:28 on dist-test-slave-7664"
15:15:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.565500 23643 fs_report.cc:383] FS layout report
15:15:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:15:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal
15:15:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal
15:15:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data/data
15:15:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:15:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:15:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:15:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:15:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:15:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:15:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:15:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:15:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:15:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:15:28.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:15:28.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.601334 23643 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:15:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.606768 23643 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:15:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.607144 23643 kserver.cc:163] Server-wide thread pool size limit: 3276
15:15:28.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.686285 23643 rpc_server.cc:307] RPC server started. Bound to: 127.22.253.125:41831
15:15:28.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.686364 23718 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.253.125:41831 every 8 connection(s)
15:15:28.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.690058 23643 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data/info.pb
15:15:28.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.695538 23541 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu as pid 23643
15:15:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.695981 23541 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal/instance
15:15:28.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.697435 23719 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:15:28.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.708323 23719 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } has no permanent_uuid. Determining permanent_uuid...
15:15:28.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.253.124@KRBTEST.COM; defaulting to no policy
15:15:28.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:15:28.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.253.124@KRBTEST.COM" created.
15:15:28.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:28 dist-test-slave-7664 krb5kdc[23553](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176528, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.125@KRBTEST.COM for kudu/127.22.253.126@KRBTEST.COM
15:15:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:15:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.253.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.124.keytab.
15:15:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.253.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.124.keytab.
15:15:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.743146 23541 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/wal
15:15:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data
15:15:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:15:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:15:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:15:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/logs
15:15:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data/info.pb
15:15:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.253.124:38783
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.253.124
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.124.keytab
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.253.124
15:15:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:15:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:15:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:15:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.253.84:37719
15:15:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.253.126:39397,127.22.253.125:41831,127.22.253.124:38783
15:15:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35517/jwks.json
15:15:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:15:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.744252 23719 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } has no permanent_uuid. Determining permanent_uuid...
15:15:28.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:28 dist-test-slave-7664 krb5kdc[23553](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176528, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.125@KRBTEST.COM for kudu/127.22.253.125@KRBTEST.COM
15:15:28.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.772795 23719 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } has no permanent_uuid. Determining permanent_uuid...
15:15:28.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:28.774828 23667 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:28.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:28.778779 23719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.828233 23719 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } attempt: 1
15:15:28.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:28.831491 23719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:28.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.899892 23719 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } attempt: 2
15:15:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:28.903981 23719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:28.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:28.995440 23719 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } attempt: 3
15:15:29.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.000564 23719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:29.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.078940 23726 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:15:29.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.079636 23726 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:15:29.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.079672 23637 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } attempt: 7
15:15:29.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.080538 23726 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:15:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:29 dist-test-slave-7664 krb5kdc[23553](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176526, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.126@KRBTEST.COM for kudu/127.22.253.125@KRBTEST.COM
15:15:29.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.106601 23637 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } has no permanent_uuid. Determining permanent_uuid...
15:15:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.113704 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:29.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.123006 23726 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:15:29.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.123276 23726 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:15:29.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.123675 23726 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:15:29.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.123972 23726 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:15:29.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.124253 23726 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:15:29.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.124545 23726 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:15:29.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.124771 23726 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35517/jwks.json
15:15:29.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.144018 23719 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } attempt: 4
15:15:29.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.147509 23719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.160495 23726 master_runner.cc:386] Master server non-default flags:
15:15:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.253.84:37719
15:15:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data
15:15:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/wal
15:15:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:15:29.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.253.126:39397,127.22.253.125:41831,127.22.253.124:38783
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.124.keytab
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.253.124
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.253.124:38783
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35517/jwks.json
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data/info.pb
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.253.124
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/logs
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 14:32:06 UTC on ccycloud.quasar-akrprb.root.comops.site
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4667498
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:15:29.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.161527 23726 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:15:29.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.162464 23726 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:15:29.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.172009 23637 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } attempt: 1
15:15:29.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.179018 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.186358 23735 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.260528 23637 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } attempt: 2
15:15:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.267294 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:29.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.352641 23637 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } attempt: 3
15:15:29.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.357118 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:29.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.446914 23719 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } attempt: 5
15:15:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.450139 23719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:29.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.497534 23637 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } attempt: 4
15:15:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.501890 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:29.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.785236 23637 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } attempt: 5
15:15:29.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.789405 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.997741 23719 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } attempt: 6
15:15:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:30.007190 23719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:30.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.341750 23637 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } attempt: 6
15:15:30.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:30.348646 23637 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.253.124:38783: Network error: Client connection negotiation failed: client connection to 127.22.253.124:38783: connect: Connection refused (error 111)
15:15:30.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.186645 23736 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:30.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:29.187178 23738 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:30.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:29.191296 23726 server_base.cc:1034] running on GCE node
15:15:30.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.398149 23726 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:15:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.400817 23726 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:15:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.402221 23726 hybrid_clock.cc:648] HybridClock initialized: now 1790176530402204 us; error 45 us; skew 500 ppm
15:15:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.402988 23726 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:15:30.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:30 dist-test-slave-7664 krb5kdc[23553](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176530, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:15:30.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.408572 23726 init.cc:377] Logged in from keytab as kudu/127.22.253.124@KRBTEST.COM (short username kudu)
15:15:30.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.413726 23726 webserver.cc:460] Webserver started at http://127.22.253.124:33471/ using document root <none> and password file <none>
15:15:30.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.414471 23726 fs_manager.cc:359] Metadata directory not provided
15:15:30.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.414685 23726 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:15:30.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.415036 23726 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:15:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.419548 23726 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data/instance:
15:15:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1739a664123140b780df17e3441e98ec"
15:15:30.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:30 on dist-test-slave-7664"
15:15:30.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.420655 23726 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/wal/instance:
15:15:30.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1739a664123140b780df17e3441e98ec"
15:15:30.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:30 on dist-test-slave-7664"
15:15:30.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.427713 23726 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
15:15:30.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.432591 23752 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:15:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.433501 23726 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
15:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.433741 23726 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data,/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/wal
15:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1739a664123140b780df17e3441e98ec"
15:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:30 on dist-test-slave-7664"
15:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.434001 23726 fs_report.cc:383] FS layout report
15:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/wal
15:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/wal
15:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data/data
15:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:15:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:15:30.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:15:30.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:15:30.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:15:30.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:15:30.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:15:30.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:15:30.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.467007 23726 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:15:30.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.472083 23726 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:15:30.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.472450 23726 kserver.cc:163] Server-wide thread pool size limit: 3276
15:15:30.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.551712 23726 rpc_server.cc:307] RPC server started. Bound to: 127.22.253.124:38783
15:15:30.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.551789 23803 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.253.124:38783 every 8 connection(s)
15:15:30.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.554335 23726 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data/info.pb
15:15:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.559022 23804 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:15:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.560022 23541 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu as pid 23726
15:15:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.560309 23541 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/wal/instance
15:15:30.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.574867 23804 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } has no permanent_uuid. Determining permanent_uuid...
15:15:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:30 dist-test-slave-7664 krb5kdc[23553](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176524, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.253.124@KRBTEST.COM
15:15:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:30 dist-test-slave-7664 krb5kdc[23553](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176530, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.124@KRBTEST.COM for kudu/127.22.253.126@KRBTEST.COM
15:15:30.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.618074 23804 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } has no permanent_uuid. Determining permanent_uuid...
15:15:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:30 dist-test-slave-7664 krb5kdc[23553](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176530, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.124@KRBTEST.COM for kudu/127.22.253.125@KRBTEST.COM
15:15:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:30 dist-test-slave-7664 krb5kdc[23553](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176524, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.253.125@KRBTEST.COM
15:15:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.646411 23804 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } has no permanent_uuid. Determining permanent_uuid...
15:15:30.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:30 dist-test-slave-7664 krb5kdc[23553](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176530, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.124@KRBTEST.COM for kudu/127.22.253.124@KRBTEST.COM
15:15:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:30 dist-test-slave-7664 krb5kdc[23553](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176524, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.253.126@KRBTEST.COM
15:15:30.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.673830 23804 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: Bootstrap starting.
15:15:30.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.677371 23541 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
15:15:30.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.680737 23804 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: Neither blocks nor log segments found. Creating new log.
15:15:30.687 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:15:30.689 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
15:15:30.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.683439 23804 log.cc:824] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: Log is configured to *not* fsync() on all Append() calls
15:15:30.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.688714 23804 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: No bootstrap required, opened a new log
15:15:30.714 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
15:15:30.715 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
15:15:30.720 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.253.126"
port: 39397
}
bound_http_address {
host: "127.22.253.126"
port: 33229
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.253.125"
port: 41831
}
bound_http_address {
host: "127.22.253.125"
port: 32975
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.253.124"
port: 38783
}
bound_http_address {
host: "127.22.253.124"
port: 33471
}
}
}
15:15:30.721 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:15:30.721 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:15:30.721 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:15:30.723 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:15:30.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.726272 23804 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } }
15:15:30.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.727048 23804 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:15:30.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.727373 23804 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1739a664123140b780df17e3441e98ec, State: Initialized, Role: FOLLOWER
15:15:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.728489 23804 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } }
15:15:30.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.732285 23815 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } } }
15:15:30.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.733176 23815 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [sys.catalog]: This master's current role is: FOLLOWER
15:15:30.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.733800 23804 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [sys.catalog]: configured and running, proceeding with master startup.
15:15:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:30 dist-test-slave-7664 krb5kdc[23553](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176530, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:15:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:15:30.762 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:15:30.764 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5cc
15:15:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:30.764706 23827 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:30.765033 23827 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:30.781 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:15:30.782 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5cc
15:15:30.785 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:15:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.761302 23726 master_runner.cc:186] Error getting master registration for 127.22.253.126:39397: OK, instance_id { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" instance_seqno: 1790176526657868 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:15:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.787261 23726 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:15:30.792 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
15:15:30.794 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:15:30.794 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:15:30.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:30 dist-test-slave-7664 krb5kdc[23553](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176530, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:15:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:15:30.823 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:15:30.823 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5cc
15:15:30.824 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:15:30.825 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.253.125:41831
15:15:30.827 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:15:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.827766 23541 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu with pid 23643
15:15:30.850 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:15:30.850 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.253.126:39397
15:15:30.850 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:15:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.851827 23541 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu with pid 23567
15:15:30.876 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:15:30.876 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.253.124:38783
15:15:30.876 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:15:30.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.877134 23541 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu with pid 23726
15:15:30.900 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:15:30.900 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.253.125:41831
15:15:30.902 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:30.903820 23541 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/logs
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data/info.pb
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.253.125:41831
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.253.125
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32975
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.125.keytab
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.253.125
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:15:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:15:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.253.84:37719
15:15:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.253.126:39397,127.22.253.125:41831,127.22.253.124:38783
15:15:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35517/jwks.json
15:15:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:15:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:31.198319 23831 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:15:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:31.198951 23831 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:15:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:31.199749 23831 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:15:31.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:31.230195 23831 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:15:31.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:31.230476 23831 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:15:31.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:31.230903 23831 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:15:31.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:31.231230 23831 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:15:31.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:31.231535 23831 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:15:31.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:31.231832 23831 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:15:31.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:31.232044 23831 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35517/jwks.json
15:15:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:31.267645 23831 master_runner.cc:386] Master server non-default flags:
15:15:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.253.84:37719
15:15:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data
15:15:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal
15:15:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:15:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.253.126:39397,127.22.253.125:41831,127.22.253.124:38783
15:15:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.125.keytab
15:15:31.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.253.125
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.253.125:41831
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35517/jwks.json
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data/info.pb
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.253.125
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32975
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/logs
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:15:31.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 14:32:06 UTC on ccycloud.quasar-akrprb.root.comops.site
15:15:31.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4667498
15:15:31.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:15:31.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:31.268967 23831 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:15:31.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:31.270196 23831 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:15:31.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:31.286904 23837 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:32.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:31.287087 23840 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:32.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:31.287783 23838 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:32.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:31.288760 23831 server_base.cc:1034] running on GCE node
15:15:32.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.506574 23831 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:15:32.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.509131 23831 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:15:32.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.510470 23831 hybrid_clock.cc:648] HybridClock initialized: now 1790176532510447 us; error 42 us; skew 500 ppm
15:15:32.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.511312 23831 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:15:32.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:32 dist-test-slave-7664 krb5kdc[23553](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176532, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:15:32.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.517269 23831 init.cc:377] Logged in from keytab as kudu/127.22.253.125@KRBTEST.COM (short username kudu)
15:15:32.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.523550 23831 webserver.cc:460] Webserver started at http://127.22.253.125:32975/ using document root <none> and password file <none>
15:15:32.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.524485 23831 fs_manager.cc:359] Metadata directory not provided
15:15:32.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.524678 23831 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:15:32.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.532678 23831 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
15:15:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.536788 23850 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:15:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.537873 23831 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:15:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.538188 23831 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data,/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal
15:15:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa26783c67ba45738bae5b848cf9aa9f"
15:15:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:28 on dist-test-slave-7664"
15:15:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.540422 23831 fs_report.cc:383] FS layout report
15:15:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:15:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal
15:15:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal
15:15:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data/data
15:15:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:15:32.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:15:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:15:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:15:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:15:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:15:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:15:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:15:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:15:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:15:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:15:32.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.601073 23831 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:15:32.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.608003 23831 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:15:32.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.608618 23831 kserver.cc:163] Server-wide thread pool size limit: 3276
15:15:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.680702 23831 rpc_server.cc:307] RPC server started. Bound to: 127.22.253.125:41831
15:15:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.680830 23902 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.253.125:41831 every 8 connection(s)
15:15:32.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.683534 23831 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/data/info.pb
15:15:32.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.690943 23541 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu as pid 23831
15:15:32.691 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:15:32.691 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.253.126:39397
15:15:32.691 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.693040 23541 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/logs
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data/info.pb
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:15:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.253.126:39397
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.253.126
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33229
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.126.keytab
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.253.126
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:15:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.253.84:37719
15:15:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.253.126:39397,127.22.253.125:41831,127.22.253.124:38783
15:15:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35517/jwks.json
15:15:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:15:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.694075 23903 sys_catalog.cc:263] Verifying existing consensus state
15:15:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260923 15:15:32.694914 23903 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:15:32.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.695356 23831 master.cc:537] Master@127.22.253.125:41831 shutting down...
15:15:32.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:32.767932 23831 master.cc:552] Master@127.22.253.125:41831 shutdown complete.
15:15:32.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:15:33.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:33.017190 23905 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:15:33.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:33.017894 23905 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:15:33.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:33.018633 23905 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:15:33.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:33.049285 23905 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:15:33.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:33.049568 23905 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:15:33.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:33.049922 23905 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:15:33.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:33.050230 23905 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:15:33.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:33.050551 23905 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:15:33.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:33.050829 23905 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:15:33.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:33.051044 23905 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35517/jwks.json
15:15:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:33.090274 23905 master_runner.cc:386] Master server non-default flags:
15:15:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.253.84:37719
15:15:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data
15:15:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal
15:15:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:15:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.253.126:39397,127.22.253.125:41831,127.22.253.124:38783
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.126.keytab
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.253.126
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.253.126:39397
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35517/jwks.json
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data/info.pb
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.253.126
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33229
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:15:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/logs
15:15:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:15:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:15:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:15:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:15:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:15:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:15:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 14:32:06 UTC on ccycloud.quasar-akrprb.root.comops.site
15:15:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4667498
15:15:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:15:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:33.091437 23905 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:15:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:33.092417 23905 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:15:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:33.110608 23914 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:33.110658 23911 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:33.111397 23912 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.314584 23913 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:15:34.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.314989 23905 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:15:34.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.318316 23905 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:15:34.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.321182 23905 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:15:34.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.322577 23905 hybrid_clock.cc:648] HybridClock initialized: now 1790176534322524 us; error 35 us; skew 500 ppm
15:15:34.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.323561 23905 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:15:34.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:34 dist-test-slave-7664 krb5kdc[23553](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176534, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:15:34.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.333981 23905 init.cc:377] Logged in from keytab as kudu/127.22.253.126@KRBTEST.COM (short username kudu)
15:15:34.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.339342 23905 webserver.cc:460] Webserver started at http://127.22.253.126:33229/ using document root <none> and password file <none>
15:15:34.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.340235 23905 fs_manager.cc:359] Metadata directory not provided
15:15:34.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.340473 23905 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:15:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.347999 23905 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
15:15:34.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.351950 23923 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:15:34.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.352941 23905 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
15:15:34.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.353262 23905 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data,/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal
15:15:34.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abb2d0917adf4fea983f3c1078c99841"
15:15:34.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:26 on dist-test-slave-7664"
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.355468 23905 fs_report.cc:383] FS layout report
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data/data
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:15:34.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:15:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.392155 23905 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:15:34.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.397619 23905 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:15:34.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.398099 23905 kserver.cc:163] Server-wide thread pool size limit: 3276
15:15:34.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.469630 23905 rpc_server.cc:307] RPC server started. Bound to: 127.22.253.126:39397
15:15:34.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.469699 23974 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.253.126:39397 every 8 connection(s)
15:15:34.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.472673 23905 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/data/info.pb
15:15:34.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.479187 23541 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu as pid 23905
15:15:34.480 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:15:34.480 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.253.124:38783
15:15:34.480 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:15:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.482273 23541 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu
15:15:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/wal
15:15:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data
15:15:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:15:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:15:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:15:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:34.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/logs
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data/info.pb
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.253.124:38783
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.253.124
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33471
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.124.keytab
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.253.124
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.253.84:37719
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.253.126:39397,127.22.253.125:41831,127.22.253.124:38783
15:15:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35517/jwks.json
15:15:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:15:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.483162 23975 sys_catalog.cc:263] Verifying existing consensus state
15:15:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260923 15:15:34.483836 23975 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:15:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.484162 23905 master.cc:537] Master@127.22.253.126:39397 shutting down...
15:15:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.546092 23905 master.cc:552] Master@127.22.253.126:39397 shutdown complete.
15:15:34.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:15:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.799973 23978 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:15:34.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.800541 23978 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:15:34.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.801263 23978 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:15:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.835790 23978 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:15:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.836082 23978 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:15:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.836483 23978 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:15:34.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.836783 23978 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:15:34.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.837141 23978 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:15:34.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.837447 23978 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:15:34.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.837675 23978 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35517/jwks.json
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.874574 23978 master_runner.cc:386] Master server non-default flags:
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.253.84:37719
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/wal
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.253.126:39397,127.22.253.125:41831,127.22.253.124:38783
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/krb5kdc/kudu_127.22.253.124.keytab
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.253.124
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.253.124:38783
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35517/jwks.json
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:15:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data/info.pb
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.253.124
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33471
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/logs
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 23 Sep 2026 14:32:06 UTC on ccycloud.quasar-akrprb.root.comops.site
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4667498
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:15:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.875795 23978 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:15:34.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:34.876880 23978 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:15:34.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.895327 23985 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.897564 23988 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:34.895613 23986 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:15:36.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:36.129680 23987 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1233 milliseconds
15:15:36.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.129788 23978 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:15:36.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.130860 23978 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:15:36.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.133635 23978 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:15:36.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.135017 23978 hybrid_clock.cc:648] HybridClock initialized: now 1790176536134969 us; error 60 us; skew 500 ppm
15:15:36.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.135874 23978 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:15:36.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:36 dist-test-slave-7664 krb5kdc[23553](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176536, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:15:36.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.141381 23978 init.cc:377] Logged in from keytab as kudu/127.22.253.124@KRBTEST.COM (short username kudu)
15:15:36.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.146288 23978 webserver.cc:460] Webserver started at http://127.22.253.124:33471/ using document root <none> and password file <none>
15:15:36.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.147153 23978 fs_manager.cc:359] Metadata directory not provided
15:15:36.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.147449 23978 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:15:36.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.154969 23978 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
15:15:36.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.159356 23998 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:15:36.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.160430 23978 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:15:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.160774 23978 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data,/tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/wal
15:15:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1739a664123140b780df17e3441e98ec"
15:15:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-23 15:15:30 on dist-test-slave-7664"
15:15:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.162882 23978 fs_report.cc:383] FS layout report
15:15:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:15:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/wal
15:15:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/wal
15:15:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data/data
15:15:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:15:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:15:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:15:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:15:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:15:36.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:15:36.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:15:36.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:15:36.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:15:36.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:15:36.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:15:36.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.198563 23978 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:15:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.203838 23978 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:15:36.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.204363 23978 kserver.cc:163] Server-wide thread pool size limit: 3276
15:15:36.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.275904 23978 rpc_server.cc:307] RPC server started. Bound to: 127.22.253.124:38783
15:15:36.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.276077 24049 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.253.124:38783 every 8 connection(s)
15:15:36.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.278632 23978 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskqZPCv2/test-tmp/mini-kudu-cluster8952886529428450047/master-2/data/info.pb
15:15:36.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.281533 23541 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu as pid 23978
15:15:36.282 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:15:36.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.291895 24050 sys_catalog.cc:263] Verifying existing consensus state
15:15:36.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.299430 24050 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: Bootstrap starting.
15:15:36.340 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:36.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.341886 24050 log.cc:824] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: Log is configured to *not* fsync() on all Append() calls
15:15:36.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.346712 24050 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:15:36.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.347627 24050 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: Bootstrap complete.
15:15:36.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.385291 24050 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } }
15:15:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.389364 24050 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:15:36.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.389787 24050 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1739a664123140b780df17e3441e98ec, State: Initialized, Role: FOLLOWER
15:15:36.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.390853 24050 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } }
15:15:36.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.403596 24054 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } } }
15:15:36.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.404412 24054 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [sys.catalog]: This master's current role is: FOLLOWER
15:15:36.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.406082 24050 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [sys.catalog]: configured and running, proceeding with master startup.
15:15:36.411 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:36.412 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.253.124:38783(127.22.253.124:38783)] connecting to peer
15:15:36.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:36.438323 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:36.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:36.438635 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:36.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:36.459419 24004 tablet.cc:2369] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:15:36.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.437829 23978 master_runner.cc:186] Error getting master registration for 127.22.253.126:39397: Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111),
15:15:36.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.463326 23978 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:15:36.481 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.482 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.483 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.483 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.483 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.484 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.485 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.253.124:38783(127.22.253.124:38783)] Successfully connected to peer
15:15:36.569 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
15:15:36.584 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.22.253.124
15:15:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:36 dist-test-slave-7664 krb5kdc[23553](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176530, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.253.124@KRBTEST.COM
15:15:36.858 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x9e90045e, L:/127.0.0.1:47716 - R:/127.22.253.124:38783] using SASL/GSSAPI
15:15:36.875 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:36.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.883383 24054 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:15:36.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.883904 24054 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } }
15:15:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.887118 24054 leader_election.cc:288] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397), aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831)
15:15:36.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:36.889863 23999 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:36.892 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:36.894 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:36.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:36.894907 23999 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831): Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:36.903 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.903 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.903 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.907 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.907 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.908 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:36.908733 24000 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397): Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111)
15:15:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.909148 24000 leader_election.cc:302] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1739a664123140b780df17e3441e98ec; no voters: aa26783c67ba45738bae5b848cf9aa9f, abb2d0917adf4fea983f3c1078c99841
15:15:36.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:36.910082 24054 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:15:36.912 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:36.928 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:36.930 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:36.930 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.930 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.931 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.934 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.934 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.935 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.938 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:36.948 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:36.949 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:36.950 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.950 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.953 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.954 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.957 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:36.988 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:36.990 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:36.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.993 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:36.995 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.995 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.996 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:36.998 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:37.008 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:37.009 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:37.010 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.010 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.011 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.015 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.015 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.016 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.016 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:37.048 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:37.050 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:37.051 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.054 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.055 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.055 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.057 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:37.188 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:37.190 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:37.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.192 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.193 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.193 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.194 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.196 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:37.208 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:37.209 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:37.210 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.210 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.211 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.212 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.214 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.215 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:37.442179 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:37.442473 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:37.728 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:37.729 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:37.730 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.731 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.731 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:37.733 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.733 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:37.737 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:38.088 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:38.089 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:38.090 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:38.090 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:38.090 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:38.093 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:38.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:38.093 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:38.095 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:38.168 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:38.169 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:38.170 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:38.170 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:38.171 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:38.173 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:38.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:38.173 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:38.176 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:38.348 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:38.350 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:38.350 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:38.350 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:38.351 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:38.352 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:38.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:38.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:38.355 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:38.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:38.445323 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:38.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:38.445567 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:38.752329 24078 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:15:38.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:38.752691 24078 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } }
15:15:38.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:38.754245 24078 leader_election.cc:288] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397), aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831)
15:15:38.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:38.760015 24000 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397): Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111)
15:15:38.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:38.760649 23999 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831): Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:38.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:38.760991 23999 leader_election.cc:302] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1739a664123140b780df17e3441e98ec; no voters: aa26783c67ba45738bae5b848cf9aa9f, abb2d0917adf4fea983f3c1078c99841
15:15:38.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:38.761456 24078 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:15:39.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:39.448235 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:39.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:39.448472 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:40.451021 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:40.451280 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:40.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:40.710271 24084 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:15:40.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:40.710687 24084 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } }
15:15:40.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:40.712163 24084 leader_election.cc:288] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397), aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831)
15:15:40.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:40.717239 23999 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831): Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:40.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:40.717721 24000 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397): Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111)
15:15:40.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:40.718012 24000 leader_election.cc:302] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1739a664123140b780df17e3441e98ec; no voters: aa26783c67ba45738bae5b848cf9aa9f, abb2d0917adf4fea983f3c1078c99841
15:15:40.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:40.718506 24084 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:15:40.808 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:40.809 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:40.810 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:40.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:40.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:40.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:40.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:40.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:40.814 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:41.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:41.453845 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:41.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:41.454123 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:41.465762 23978 master_runner.cc:186] Error getting master registration for 127.22.253.126:39397: Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111),
15:15:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:41.466238 23978 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:15:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:42.456789 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:42.457046 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:43.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:43.460191 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:43.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:43.460467 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:44.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:44.462939 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:44.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:44.463203 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:44.548 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:44.549 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:44.550 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:44.550 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:44.551 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:44.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:44.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:44.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:44.554 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:44.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:44.823712 24096 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:15:44.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:44.824640 24096 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } }
15:15:44.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:44.827009 24096 leader_election.cc:288] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397), aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831)
15:15:44.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:44.829741 24000 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111) [suppressed 5 similar messages]
15:15:44.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:44.832983 23999 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831): Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:44.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:44.833408 24000 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397): Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111)
15:15:44.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:44.834383 24000 leader_election.cc:302] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1739a664123140b780df17e3441e98ec; no voters: aa26783c67ba45738bae5b848cf9aa9f, abb2d0917adf4fea983f3c1078c99841
15:15:44.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:44.835098 24096 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:15:45.148 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:45.149 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:45.150 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:45.150 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:45.151 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:45.152 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:45.152 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:45.153 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:45.154 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:45.466054 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:45.466315 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:46 dist-test-slave-7664 krb5kdc[23553](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176546, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:15:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:46.144645 23994 init.cc:295] Successfully reacquired a new kerberos TGT
15:15:46.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:46.469226 23978 master_runner.cc:186] Error getting master registration for 127.22.253.126:39397: Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111),
15:15:46.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:46.469611 23978 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:15:46.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:46.469719 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:46.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:46.470003 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:47.108 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:47.109 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:47.111 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:47.111 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:47.112 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:47.112 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:47.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:47.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:47.115 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:47.348 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:47.350 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:47.351 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:47.352 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:47.352 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:47.353 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:47.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:47.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:47.355 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:47.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:47.472838 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:47.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:47.473106 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:47.511595 24101 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:15:47.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:47.511942 24101 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } }
15:15:47.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:47.513640 24101 leader_election.cc:288] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397), aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831)
15:15:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:47.518831 23999 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831): Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:47.519511 24000 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397): Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111)
15:15:47.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:47.520081 24000 leader_election.cc:302] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1739a664123140b780df17e3441e98ec; no voters: aa26783c67ba45738bae5b848cf9aa9f, abb2d0917adf4fea983f3c1078c99841
15:15:47.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:47.520833 24101 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:15:48.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:48.475978 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:48.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:48.476246 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:49.388 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:49.389 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:49.391 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:49.391 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:49.391 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:49.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:49.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:49.392 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:49.395 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:49.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:49.479014 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:49.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:49.479404 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:50.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:50.482213 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:50.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:50.482488 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:51.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:51.471875 23978 master_runner.cc:186] Error getting master registration for 127.22.253.126:39397: Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111),
15:15:51.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:51.472225 23978 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:15:51.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:51.485080 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:51.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:51.485289 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:51.808 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:51.809 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:51.810 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:51.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:51.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:51.813 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:51.813 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:51.813 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:51.815 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:52.208 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:52.209 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:52.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:52.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:52.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:52.210 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:52.210 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:52.211 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:52.213 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:52.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:52.487655 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:52.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:52.487879 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:52.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:52.511686 24112 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:15:52.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:52.512064 24112 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } }
15:15:52.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:52.513563 24112 leader_election.cc:288] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397), aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831)
15:15:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:52.515544 24000 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111) [suppressed 3 similar messages]
15:15:52.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:52.518244 23999 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831): Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:52.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:52.518630 24000 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397): Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111)
15:15:52.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:52.518997 24000 leader_election.cc:302] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1739a664123140b780df17e3441e98ec; no voters: aa26783c67ba45738bae5b848cf9aa9f, abb2d0917adf4fea983f3c1078c99841
15:15:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:52.519711 24112 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:15:53.128 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:53.129 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:53.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:53.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:53.130 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:53.130 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:53.130 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:53.131 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:53.133 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:53.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:53.490675 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:53.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:53.490904 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:54.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:54.493786 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:54.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:54.494011 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:54.748 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:54.749 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:54.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:54.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:54.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:54.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:54.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:54.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:54.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:54.768 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:54.768 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:54.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:54.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:54.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:54.770 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:54.770 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:54.771 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:54.773 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:55.497917 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:55.498221 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 23 15:15:56 dist-test-slave-7664 krb5kdc[23553](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790176556, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.253.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:15:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:56.146854 23994 init.cc:295] Successfully reacquired a new kerberos TGT
15:15:56.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:56.475236 23978 master_runner.cc:186] Error getting master registration for 127.22.253.126:39397: Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111),
15:15:56.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:56.475692 23978 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:15:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:56.501098 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:56.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:56.501348 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:56.531831 24122 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:15:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:56.532320 24122 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } }
15:15:56.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:56.534226 24122 leader_election.cc:288] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397), aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831)
15:15:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:56.538533 24000 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397): Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111)
15:15:56.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:56.539090 23999 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831): Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:15:56.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:56.539484 23999 leader_election.cc:302] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1739a664123140b780df17e3441e98ec; no voters: aa26783c67ba45738bae5b848cf9aa9f, abb2d0917adf4fea983f3c1078c99841
15:15:56.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:15:56.540099 24122 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:15:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:57.504120 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:57.504349 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:57.968 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:15:57.969 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:15:57.970 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:57.970 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:57.970 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:15:57.971 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:57.971 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:57.971 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:15:57.973 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:15:58.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:58.507117 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:58.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:58.507386 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:58.588189 23383 debug-util.cc:398] Leaking SignalData structure 0x7b08000bdf00 after lost signal to thread 23257
15:15:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:58.589486 23383 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8040 after lost signal to thread 23386
15:15:59.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:59.510265 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:15:59.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:59.510550 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:15:59.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:59.571547 23526 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5400 after lost signal to thread 23398
15:15:59.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:15:59.572398 23526 debug-util.cc:398] Leaking SignalData structure 0x7b08000af440 after lost signal to thread 23529
15:16:00.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:00.145722 24128 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:16:00.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:00.146123 24128 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "abb2d0917adf4fea983f3c1078c99841" member_type: VOTER last_known_addr { host: "127.22.253.126" port: 39397 } } peers { permanent_uuid: "aa26783c67ba45738bae5b848cf9aa9f" member_type: VOTER last_known_addr { host: "127.22.253.125" port: 41831 } } peers { permanent_uuid: "1739a664123140b780df17e3441e98ec" member_type: VOTER last_known_addr { host: "127.22.253.124" port: 38783 } }
15:16:00.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:00.147516 24128 leader_election.cc:288] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397), aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831)
15:16:00.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:00.149194 24000 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111) [suppressed 3 similar messages]
15:16:00.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:00.151729 23999 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer aa26783c67ba45738bae5b848cf9aa9f (127.22.253.125:41831): Network error: Client connection negotiation failed: client connection to 127.22.253.125:41831: connect: Connection refused (error 111)
15:16:00.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:00.152154 24000 leader_election.cc:334] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer abb2d0917adf4fea983f3c1078c99841 (127.22.253.126:39397): Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111)
15:16:00.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:00.152588 24000 leader_election.cc:302] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1739a664123140b780df17e3441e98ec; no voters: aa26783c67ba45738bae5b848cf9aa9f, abb2d0917adf4fea983f3c1078c99841
15:16:00.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:00.153250 24128 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:16:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:00.514187 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:16:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:00.514410 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:16:00.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:00.920115 23038 debug-util.cc:398] Leaking SignalData structure 0x7b080007d8a0 after lost signal to thread 22970
15:16:00.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:00.920854 23038 debug-util.cc:398] Leaking SignalData structure 0x7b08000bcae0 after lost signal to thread 23041
15:16:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:01.478180 23978 master_runner.cc:186] Error getting master registration for 127.22.253.126:39397: Network error: Client connection negotiation failed: client connection to 127.22.253.126:39397: connect: Connection refused (error 111),
15:16:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:01.478601 23978 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:16:01.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:01.517235 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:16:01.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:01.517524 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:16:01.528 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:16:01.529 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:16:01.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:16:01.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:16:01.530 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:16:01.530 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:16:01.530 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:16:01.530 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:16:01.534 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:16:02.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:02.520412 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:16:02.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:02.520680 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:16:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:03.523862 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:16:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:03.524186 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:16:04.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:04.526825 24067 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1739a664123140b780df17e3441e98ec: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:16:04.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260923 15:16:04.527060 24067 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:16:04.648 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.125:41831(127.22.253.125:41831)] connecting to peer
15:16:04.649 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.253.126:39397(127.22.253.126:39397)] connecting to peer
15:16:04.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:16:04.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.253.125:41831(127.22.253.125:41831)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:16:04.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.125:41831: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831
15:16:04.650 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:16:04.650 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.126:39397(127.22.253.126:39397)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:16:04.651 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.253.126:39397: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
15:16:04.653 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783; will retry
15:16:04.657 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28366), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [190ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [191ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [579ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [583ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [599ms] refreshing cache from master, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [602ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [603ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [611ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [615ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [619ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [620ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [639ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [645ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [658ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [661ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [700ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [703ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [705ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [706ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [719ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [722ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [724ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [755ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [760ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [763ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [764ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [899ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [901ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [904ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [918ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [921ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [923ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1435ms] refreshing cache from master, [1435ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1439ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1441ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1444ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1445ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1795ms] refreshing cache from master, [1795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1798ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1801ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1803ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1803ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1875ms] refreshing cache from master, [1875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1879ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1883ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1884ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [2056ms] refreshing cache from master, [2056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [2057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [2058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [2058ms] too many traces: truncated at 100 traces, deferred=Deferred@1265971176(state=PENDING, result=null, callback=(continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901) -> (continuation of Deferred@1613377914 after retry RPC after error@1324799239), errback=(continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901) -> (continuation of Deferred@1613377914 after retry RPC after error@1324799239)))
15:16:04.661 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.253.125:41831(127.22.253.125:41831)] disconnecting while in state TERMINATED
15:16:04.662 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.253.126:39397(127.22.253.126:39397)] disconnecting while in state TERMINATED
15:16:04.662 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.253.124:38783(127.22.253.124:38783)] disconnecting while in state READY
15:16:04.666 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:16:04.668 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.253.124:38783(127.22.253.124:38783)] handling channelInactive
15:16:04.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.253.124:38783(127.22.253.124:38783)] cleaning up while in state READY due to: connection closed
15:16:04.670 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:16:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:04.673113 23541 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu with pid 23978
15:16:05.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-23T15:16:05Z chronyd exiting
15:16:06.815 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:16:06.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:06.817108 22808 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu with pid 23093
15:16:06.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:06.841570 22808 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu with pid 23251
15:16:06.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:06.868793 22808 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu with pid 23397
15:16:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:06.896235 22808 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu with pid 22820
15:16:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:06.924993 22808 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu with pid 22890
15:16:06.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260923 15:16:06.953172 22808 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskqZPCv2/build/tsan/bin/kudu with pid 22965
15:16:08.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-23T15:16:08Z chronyd exiting
15:16:09.092 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28366), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [190ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [191ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [579ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [583ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [599ms] refreshing cache from master, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [602ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [603ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [611ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [615ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [619ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [620ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [639ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [645ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [658ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [661ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [700ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [703ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [705ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [706ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [719ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [722ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [724ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [755ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [760ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [763ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [764ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [899ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [901ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [904ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [918ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [921ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [923ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1435ms] refreshing cache from master, [1435ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1439ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1441ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1444ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1445ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1795ms] refreshing cache from master, [1795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1798ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1801ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1803ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1803ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1875ms] refreshing cache from master, [1875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1879ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1883ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1884ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [2056ms] refreshing cache from master, [2056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [2057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [2058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [2058ms] too many traces: truncated at 100 traces, deferred=Deferred@1265971176(state=PENDING, result=null, callback=(continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901) -> (continuation of Deferred@1613377914 after retry RPC after error@1324799239), errback=(continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901) -> (continuation of Deferred@1613377914 after retry RPC after error@1324799239)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.253.126:39397
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28366), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [190ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [191ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [579ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [583ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [599ms] refreshing cache from master, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [602ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [603ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [611ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [615ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [619ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [620ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [639ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [645ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [658ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [661ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [700ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [703ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [705ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [706ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [719ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [722ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [724ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [755ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [760ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [763ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [764ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [899ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [901ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [904ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [918ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [921ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [923ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1435ms] refreshing cache from master, [1435ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1439ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1441ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1444ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1445ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1795ms] refreshing cache from master, [1795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1798ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1801ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1803ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1803ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1875ms] refreshing cache from master, [1875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1879ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1883ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1884ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [2056ms] refreshing cache from master, [2056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [2057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [2058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [2058ms] too many traces: truncated at 100 traces, deferred=Deferred@1265971176(state=PENDING, result=null, callback=(continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901) -> (continuation of Deferred@1613377914 after retry RPC after error@1324799239), errback=(continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901) -> (continuation of Deferred@1613377914 after retry RPC after error@1324799239)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.253.126:39397
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
15:16:09.631 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
15:16:09.631 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28366), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [190ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [191ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [579ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [583ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [599ms] refreshing cache from master, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [602ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [603ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [611ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [615ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [619ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [620ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [639ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [645ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [658ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [661ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [700ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [703ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [705ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [706ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [719ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [722ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [724ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [755ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [760ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [763ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [764ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [899ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [901ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [904ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [918ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [921ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [923ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1435ms] refreshing cache from master, [1435ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1439ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1441ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1444ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1445ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1795ms] refreshing cache from master, [1795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1798ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1801ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1803ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1803ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1875ms] refreshing cache from master, [1875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1879ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1883ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1884ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [2056ms] refreshing cache from master, [2056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [2057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [2058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [2058ms] too many traces: truncated at 100 traces, deferred=Deferred@1265971176(state=PENDING, result=null, callback=(continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901) -> (continuation of Deferred@1613377914 after retry RPC after error@1324799239), errback=(continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901) -> (continuation of Deferred@1613377914 after retry RPC after error@1324799239)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.253.126:39397
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28366), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [190ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [191ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [579ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [583ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [599ms] refreshing cache from master, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [602ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [603ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [611ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [615ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [619ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [620ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [639ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [645ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [658ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [661ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [700ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [703ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [705ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [706ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [719ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [722ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [723ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [724ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [755ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [760ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [763ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [764ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [896ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [899ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [901ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [904ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [915ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [918ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [921ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [923ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1435ms] refreshing cache from master, [1435ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1438ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1439ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1441ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1444ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1445ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1795ms] refreshing cache from master, [1795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1798ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1801ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1803ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1803ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1875ms] refreshing cache from master, [1875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [1879ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.125:41831: Network error: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.126:39397: Network error: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [1883ms] Sub RPC ConnectToMaster: received response from server master-127.22.253.124:38783: OK, [1884ms] delaying RPC due to: Service unavailable: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397, [2056ms] refreshing cache from master, [2056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.125:41831, [2057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.126:39397, [2058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.253.124:38783, [2058ms] too many traces: truncated at 100 traces, deferred=Deferred@1265971176(state=PENDING, result=null, callback=(continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901) -> (continuation of Deferred@1613377914 after retry RPC after error@1324799239), errback=(continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901) -> (continuation of Deferred@1613377914 after retry RPC after error@1324799239)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.253.125:41831,127.22.253.126:39397,127.22.253.124:38783) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.125:41831(127.22.253.125:41831): Connection refused: /127.22.253.125:41831,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.253.126:39397(127.22.253.126:39397): Connection refused: /127.22.253.126:39397
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.253.126:39397
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)