01:57:56.556 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
01:57:56.567 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskaDlrHb/test-tmp
01:57:56.569 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:57:56.573 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskaDlrHb/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:57:56.780 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
01:57:56.960 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:57:56.961 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:57:56.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:56.966420 25568 mini_oidc.cc:152] Starting JWKS server
01:57:56.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:56.970649 25568 webserver.cc:460] Webserver started at http://127.0.0.1:43439/ using document root <none> and password file <none>
01:57:56.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:56.971251 25568 mini_oidc.cc:189] Starting OIDC Discovery server
01:57:56.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:56.973387 25568 webserver.cc:460] Webserver started at http://127.0.0.1:36915/ using document root <none> and password file <none>
01:57:57.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T01:57:57Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:57:57.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T01:57:57Z Disabled control of system clock
01:57:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:57.019733 25568 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:57:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:57:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/wal
01:57:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/data
01:57:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:57:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:57:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:57:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/logs
01:57:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/data/info.pb
01:57:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:57:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:57:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:57:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:57:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:57.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:57:57.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:57:57.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:57.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:57.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.248.62:42099
01:57:57.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.248.62
01:57:57.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:57.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.248.20:39725
01:57:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.248.62:42099,127.24.248.61:34905,127.24.248.60:36249
01:57:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43439/jwks.json with env {}
01:57:57.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:57.311167 25580 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:57:57.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:57.311733 25580 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:57:57.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:57.312368 25580 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:57:57.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:57.343032 25580 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:57:57.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:57.343326 25580 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:57:57.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:57.343519 25580 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:57:57.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:57.343695 25580 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:57:57.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:57.343973 25580 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:57:57.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:57.344197 25580 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43439/jwks.json
01:57:57.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:57.378057 25580 master_runner.cc:386] Master server non-default flags:
01:57:57.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:57.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.248.20:39725
01:57:57.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:57.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:57.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/data
01:57:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/wal
01:57:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.248.62:42099,127.24.248.61:34905,127.24.248.60:36249
01:57:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.248.62:42099
01:57:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:57:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43439/jwks.json
01:57:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/data/info.pb
01:57:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.248.62
01:57:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:57:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:57:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:57:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/logs
01:57:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:57:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:57:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:57:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:57:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:57:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:57:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
01:57:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
01:57:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:57:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:57.379376 25580 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:57:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:57.381500 25580 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:57:57.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:57.400624 25586 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:58.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:57.403074 25580 server_base.cc:1034] running on GCE node
01:57:58.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:57.401418 25587 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:58.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:57.400892 25589 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:58.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.572990 25580 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:57:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.575587 25580 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:57:58.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.576928 25580 hybrid_clock.cc:648] HybridClock initialized: now 1790387878576886 us; error 53 us; skew 500 ppm
01:57:58.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.577806 25580 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:57:58.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.588243 25580 webserver.cc:460] Webserver started at http://127.24.248.62:46357/ using document root <none> and password file <none>
01:57:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.589260 25580 fs_manager.cc:359] Metadata directory not provided
01:57:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.589443 25580 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:57:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.589835 25580 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:57:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.594588 25580 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/data/instance:
01:57:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad1ef6621d884f90900c0d1ce63d4483"
01:57:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:57:58 on dist-test-slave-52vw"
01:57:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.595768 25580 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/wal/instance:
01:57:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad1ef6621d884f90900c0d1ce63d4483"
01:57:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:57:58 on dist-test-slave-52vw"
01:57:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.602860 25580 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.001s sys 0.008s
01:57:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.608325 25596 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:57:58.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.609354 25580 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
01:57:58.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.609700 25580 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/data,/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/wal
01:57:58.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad1ef6621d884f90900c0d1ce63d4483"
01:57:58.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:57:58 on dist-test-slave-52vw"
01:57:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.610086 25580 fs_report.cc:383] FS layout report
01:57:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:57:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/wal
01:57:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/wal
01:57:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/data/data
01:57:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:57:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:57:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:57:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:57:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:57:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:57:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:57:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:57:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:57:58.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:57:58.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:57:58.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.645345 25580 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:57:58.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.646963 25580 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:57:58.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.647470 25580 kserver.cc:163] Server-wide thread pool size limit: 3276
01:57:58.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.715975 25580 rpc_server.cc:307] RPC server started. Bound to: 127.24.248.62:42099
01:57:58.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.716071 25647 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.248.62:42099 every 8 connection(s)
01:57:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.718854 25580 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/data/info.pb
01:57:58.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.723908 25648 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:57:58.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.728519 25568 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu as pid 25580
01:57:58.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.729008 25568 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-0/wal/instance
01:57:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.732501 25568 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:57:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:57:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/wal
01:57:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/data
01:57:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:57:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:57:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:57:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:58.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/logs
01:57:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/data/info.pb
01:57:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:57:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:57:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:57:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:57:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:58.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:58.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:58.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:57:58.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:57:58.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:58.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:58.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.248.61:34905
01:57:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.248.61
01:57:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.248.20:39725
01:57:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.248.62:42099,127.24.248.61:34905,127.24.248.60:36249
01:57:58.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:58.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43439/jwks.json with env {}
01:57:58.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.735075 25648 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } has no permanent_uuid. Determining permanent_uuid...
01:57:58.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.763015 25648 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } has no permanent_uuid. Determining permanent_uuid...
01:57:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:58.765329 25600 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.248.61:34905: connect: Connection refused (error 111)
01:57:58.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:58.769358 25648 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.61:34905: Network error: Client connection negotiation failed: client connection to 127.24.248.61:34905: connect: Connection refused (error 111)
01:57:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.818921 25648 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } attempt: 1
01:57:58.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:58.823450 25648 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.61:34905: Network error: Client connection negotiation failed: client connection to 127.24.248.61:34905: connect: Connection refused (error 111)
01:57:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.891853 25648 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } attempt: 2
01:57:58.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:58.895597 25648 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.61:34905: Network error: Client connection negotiation failed: client connection to 127.24.248.61:34905: connect: Connection refused (error 111)
01:57:58.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:58.987021 25648 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } attempt: 3
01:57:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:58.990486 25648 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.61:34905: Network error: Client connection negotiation failed: client connection to 127.24.248.61:34905: connect: Connection refused (error 111)
01:57:59.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.033249 25650 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:57:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.033804 25650 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:57:59.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.034441 25650 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:57:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.065106 25650 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:57:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.065372 25650 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:57:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.065725 25650 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:57:59.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.066039 25650 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:57:59.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.066321 25650 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:57:59.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.066570 25650 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43439/jwks.json
01:57:59.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:59.101749 25650 master_runner.cc:386] Master server non-default flags:
01:57:59.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:57:59.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.248.20:39725
01:57:59.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:57:59.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:57:59.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/data
01:57:59.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/wal
01:57:59.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:57:59.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.248.62:42099,127.24.248.61:34905,127.24.248.60:36249
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.248.61:34905
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43439/jwks.json
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/data/info.pb
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.248.61
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:57:59.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:57:59.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:57:59.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:57:59.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:57:59.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/logs
01:57:59.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:57:59.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:57:59.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:57:59.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:57:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:57:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:57:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:57:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
01:57:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
01:57:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:57:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:59.102910 25650 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:57:59.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:59.105021 25650 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:57:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.121984 25660 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:57:59.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:59.133946 25648 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } attempt: 4
01:57:59.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.137351 25648 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.61:34905: Network error: Client connection negotiation failed: client connection to 127.24.248.61:34905: connect: Connection refused (error 111)
01:57:59.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:59.436951 25648 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } attempt: 5
01:57:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.440426 25648 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.61:34905: Network error: Client connection negotiation failed: client connection to 127.24.248.61:34905: connect: Connection refused (error 111)
01:57:59.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:59.988116 25648 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } attempt: 6
01:57:59.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.994992 25648 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.61:34905: Network error: Client connection negotiation failed: client connection to 127.24.248.61:34905: connect: Connection refused (error 111)
01:58:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.122809 25659 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:57:59.122083 25662 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:57:59.123850 25650 server_base.cc:1034] running on GCE node
01:58:00.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.304159 25650 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:58:00.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.306988 25650 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:58:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.310555 25650 hybrid_clock.cc:648] HybridClock initialized: now 1790387880309923 us; error 639 us; skew 500 ppm
01:58:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.311584 25650 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:58:00.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.318197 25650 webserver.cc:460] Webserver started at http://127.24.248.61:45183/ using document root <none> and password file <none>
01:58:00.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.319288 25650 fs_manager.cc:359] Metadata directory not provided
01:58:00.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.319551 25650 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:58:00.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.320118 25650 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:58:00.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.325094 25650 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/data/instance:
01:58:00.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3328fd2d10a1469e87124aa364669240"
01:58:00.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:00 on dist-test-slave-52vw"
01:58:00.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.326352 25650 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/wal/instance:
01:58:00.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3328fd2d10a1469e87124aa364669240"
01:58:00.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:00 on dist-test-slave-52vw"
01:58:00.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.333559 25650 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.003s sys 0.005s
01:58:00.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.338923 25671 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:58:00.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.340058 25650 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
01:58:00.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.340330 25650 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/data,/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/wal
01:58:00.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3328fd2d10a1469e87124aa364669240"
01:58:00.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:00 on dist-test-slave-52vw"
01:58:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.340621 25650 fs_report.cc:383] FS layout report
01:58:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:58:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/wal
01:58:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/wal
01:58:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/data/data
01:58:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:58:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:58:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:58:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:58:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:58:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:58:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:58:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:58:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:58:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:58:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:58:00.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.375003 25650 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:58:00.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.376421 25650 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:58:00.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.376839 25650 kserver.cc:163] Server-wide thread pool size limit: 3276
01:58:00.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.444005 25650 rpc_server.cc:307] RPC server started. Bound to: 127.24.248.61:34905
01:58:00.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.444131 25722 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.248.61:34905 every 8 connection(s)
01:58:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.446597 25650 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/data/info.pb
01:58:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.450289 25568 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu as pid 25650
01:58:00.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.450745 25568 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-1/wal/instance
01:58:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.452277 25723 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.453706 25568 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/wal
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/data
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/logs
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/data/info.pb
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:00.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.248.60:36249
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.248.60
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.248.20:39725
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.248.62:42099,127.24.248.61:34905,127.24.248.60:36249
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:00.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43439/jwks.json with env {}
01:58:00.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.465727 25723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } has no permanent_uuid. Determining permanent_uuid...
01:58:00.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.491849 25723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } has no permanent_uuid. Determining permanent_uuid...
01:58:00.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.508075 25723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } has no permanent_uuid. Determining permanent_uuid...
01:58:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.510273 25675 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.514314 25723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.60:36249: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.563778 25723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } attempt: 1
01:58:00.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.567432 25723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.60:36249: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:00.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.635818 25723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } attempt: 2
01:58:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.640130 25723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.60:36249: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.731580 25723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } attempt: 3
01:58:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.735895 25723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.60:36249: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:00.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.763446 25725 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:58:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.764601 25725 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:58:00.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.765396 25725 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:58:00.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.801470 25725 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:58:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.801828 25725 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:58:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.802183 25725 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:58:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.802548 25725 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:58:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.802899 25725 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:58:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.803154 25725 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43439/jwks.json
01:58:00.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.839386 25725 master_runner.cc:386] Master server non-default flags:
01:58:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.248.20:39725
01:58:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/data
01:58:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/wal
01:58:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.248.62:42099,127.24.248.61:34905,127.24.248.60:36249
01:58:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.248.60:36249
01:58:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43439/jwks.json
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/data/info.pb
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.248.60
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/logs
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:58:00.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:58:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:58:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:58:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:58:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:58:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
01:58:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
01:58:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:58:00.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.840909 25725 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:58:00.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.842989 25725 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:58:00.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.860905 25735 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:00.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.879274 25723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } attempt: 4
01:58:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.883392 25723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.60:36249: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:01.055714 25648 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } attempt: 7
01:58:01.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:01.078727 25648 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } has no permanent_uuid. Determining permanent_uuid...
01:58:01.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:01.083668 25648 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.60:36249: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:01.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:01.142064 25648 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } attempt: 1
01:58:01.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:01.146148 25648 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.60:36249: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:01.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:01.182835 25723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } attempt: 5
01:58:01.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:01.187039 25723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.60:36249: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:01.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:01.227491 25648 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } attempt: 2
01:58:01.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:01.231712 25648 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.60:36249: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:01.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:01.317221 25648 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } attempt: 3
01:58:01.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:01.321290 25648 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.60:36249: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:01.461870 25648 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } attempt: 4
01:58:01.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:01.467599 25648 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.60:36249: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:01.734401 25723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } attempt: 6
01:58:01.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:01.742017 25723 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.60:36249: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:01.751298 25648 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } attempt: 5
01:58:01.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:01.756213 25648 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.248.60:36249: Network error: Client connection negotiation failed: client connection to 127.24.248.60:36249: connect: Connection refused (error 111)
01:58:02.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.861001 25736 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:02.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:00.861296 25738 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:02.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:00.863140 25725 server_base.cc:1034] running on GCE node
01:58:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.064471 25725 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:58:02.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.067672 25725 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:58:02.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.069075 25725 hybrid_clock.cc:648] HybridClock initialized: now 1790387882069025 us; error 72 us; skew 500 ppm
01:58:02.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.069864 25725 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:58:02.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.076783 25725 webserver.cc:460] Webserver started at http://127.24.248.60:32889/ using document root <none> and password file <none>
01:58:02.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.077665 25725 fs_manager.cc:359] Metadata directory not provided
01:58:02.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.077844 25725 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:58:02.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.078205 25725 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:58:02.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.084920 25725 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/data/instance:
01:58:02.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4f1d98b13394b04986fe5948ab45ac8"
01:58:02.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:02 on dist-test-slave-52vw"
01:58:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.086390 25725 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/wal/instance:
01:58:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4f1d98b13394b04986fe5948ab45ac8"
01:58:02.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:02 on dist-test-slave-52vw"
01:58:02.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.095638 25725 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
01:58:02.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.102909 25750 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:58:02.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.104539 25725 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
01:58:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.104969 25725 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/data,/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/wal
01:58:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4f1d98b13394b04986fe5948ab45ac8"
01:58:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:02 on dist-test-slave-52vw"
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.105407 25725 fs_report.cc:383] FS layout report
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/wal
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/wal
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/data/data
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:58:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:58:02.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.138329 25725 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:58:02.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.139724 25725 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:58:02.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.140105 25725 kserver.cc:163] Server-wide thread pool size limit: 3276
01:58:02.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.207392 25725 rpc_server.cc:307] RPC server started. Bound to: 127.24.248.60:36249
01:58:02.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.207485 25801 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.248.60:36249 every 8 connection(s)
01:58:02.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.209991 25725 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/data/info.pb
01:58:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.214170 25568 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu as pid 25725
01:58:02.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.214627 25568 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/master-2/wal/instance
01:58:02.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.215497 25802 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:58:02.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.234210 25802 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } has no permanent_uuid. Determining permanent_uuid...
01:58:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.259817 25802 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } has no permanent_uuid. Determining permanent_uuid...
01:58:02.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.271427 25802 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } has no permanent_uuid. Determining permanent_uuid...
01:58:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.288805 25802 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8: Bootstrap starting.
01:58:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.295375 25802 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8: Neither blocks nor log segments found. Creating new log.
01:58:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.297235 25802 log.cc:824] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8: Log is configured to *not* fsync() on all Append() calls
01:58:02.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.301554 25802 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8: No bootstrap required, opened a new log
01:58:02.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.308606 25648 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } attempt: 6
01:58:02.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.328519 25648 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483: Bootstrap starting.
01:58:02.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.329038 25802 raft_consensus.cc:348] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } }
01:58:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.330188 25802 raft_consensus.cc:374] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:58:02.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.330485 25802 raft_consensus.cc:729] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b4f1d98b13394b04986fe5948ab45ac8, State: Initialized, Role: FOLLOWER
01:58:02.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.331714 25802 consensus_queue.cc:260] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } }
01:58:02.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.334542 25812 sys_catalog.cc:455] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } } }
01:58:02.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.335431 25812 sys_catalog.cc:458] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [sys.catalog]: This master's current role is: FOLLOWER
01:58:02.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.335767 25648 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483: Neither blocks nor log segments found. Creating new log.
01:58:02.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.335878 25802 sys_catalog.cc:564] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [sys.catalog]: configured and running, proceeding with master startup.
01:58:02.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.338068 25648 log.cc:824] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483: Log is configured to *not* fsync() on all Append() calls
01:58:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.342446 25648 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483: No bootstrap required, opened a new log
01:58:02.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.370497 25648 raft_consensus.cc:348] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } }
01:58:02.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.371796 25648 raft_consensus.cc:374] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:58:02.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:02.372618 25824 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:02.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.372732 25648 raft_consensus.cc:729] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ad1ef6621d884f90900c0d1ce63d4483, State: Initialized, Role: FOLLOWER
01:58:02.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:02.373332 25824 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:02.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.373805 25648 consensus_queue.cc:260] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } }
01:58:02.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.376881 25826 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } } }
01:58:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.378726 25648 sys_catalog.cc:564] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [sys.catalog]: configured and running, proceeding with master startup.
01:58:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.386245 25826 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [sys.catalog]: This master's current role is: FOLLOWER
01:58:02.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.361855 25725 master_runner.cc:186] Error getting master registration for 127.24.248.62:42099: OK, instance_id { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" instance_seqno: 1790387878697407 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:58:02.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.393178 25725 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:58:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:02.394989 25756 tablet.cc:2369] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:58:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:02.405974 25602 tablet.cc:2369] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:58:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:02.418774 25837 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:02.424440 25837 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:02.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.417642 25580 master_runner.cc:186] Error getting master registration for 127.24.248.61:34905: OK, instance_id { permanent_uuid: "3328fd2d10a1469e87124aa364669240" instance_seqno: 1790387880425649 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:58:02.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.456095 25580 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:58:02.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.479977 25812 raft_consensus.cc:482] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:58:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.480500 25812 raft_consensus.cc:504] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } }
01:58:02.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.483067 25812 leader_election.cc:288] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ad1ef6621d884f90900c0d1ce63d4483 (127.24.248.62:42099), 3328fd2d10a1469e87124aa364669240 (127.24.248.61:34905)
01:58:02.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.484251 25623 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b4f1d98b13394b04986fe5948ab45ac8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ad1ef6621d884f90900c0d1ce63d4483" is_pre_election: true
01:58:02.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.485188 25623 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b4f1d98b13394b04986fe5948ab45ac8 in term 0.
01:58:02.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.486562 25751 leader_election.cc:302] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ad1ef6621d884f90900c0d1ce63d4483, b4f1d98b13394b04986fe5948ab45ac8; no voters:
01:58:02.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.487524 25812 raft_consensus.cc:2781] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [term 0 FOLLOWER]: Leader pre-election won for term 1
01:58:02.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.487850 25812 raft_consensus.cc:482] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:58:02.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.488185 25812 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [term 0 FOLLOWER]: Advancing to term 1
01:58:02.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.494778 25812 raft_consensus.cc:504] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } }
01:58:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.499547 25623 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b4f1d98b13394b04986fe5948ab45ac8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ad1ef6621d884f90900c0d1ce63d4483"
01:58:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.500129 25623 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [term 0 FOLLOWER]: Advancing to term 1
01:58:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.500867 25812 leader_election.cc:288] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [CANDIDATE]: Term 1 election: Requested vote from peers ad1ef6621d884f90900c0d1ce63d4483 (127.24.248.62:42099), 3328fd2d10a1469e87124aa364669240 (127.24.248.61:34905)
01:58:02.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.507381 25623 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b4f1d98b13394b04986fe5948ab45ac8 in term 1.
01:58:02.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.509359 25751 leader_election.cc:302] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ad1ef6621d884f90900c0d1ce63d4483, b4f1d98b13394b04986fe5948ab45ac8; no voters:
01:58:02.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.513563 25812 raft_consensus.cc:2781] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [term 1 FOLLOWER]: Leader election won for term 1
01:58:02.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.530051 25812 raft_consensus.cc:686] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [term 1 LEADER]: Becoming Leader. State: Replica: b4f1d98b13394b04986fe5948ab45ac8, State: Running, Role: LEADER
01:58:02.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.531487 25812 consensus_queue.cc:237] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } }
01:58:02.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.547978 25839 sys_catalog.cc:455] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b4f1d98b13394b04986fe5948ab45ac8. Latest consensus state: current_term: 1 leader_uuid: "b4f1d98b13394b04986fe5948ab45ac8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } } }
01:58:02.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.550446 25839 sys_catalog.cc:458] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [sys.catalog]: This master's current role is: LEADER
01:58:02.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.555142 25841 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:58:02.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.562438 25841 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:58:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.587045 25623 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [term 1 FOLLOWER]: Refusing update from remote peer b4f1d98b13394b04986fe5948ab45ac8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:58:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.593192 25839 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:58:02.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.617659 25826 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b4f1d98b13394b04986fe5948ab45ac8. Latest consensus state: current_term: 1 leader_uuid: "b4f1d98b13394b04986fe5948ab45ac8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } } }
01:58:02.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.618749 25826 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [sys.catalog]: This master's current role is: FOLLOWER
01:58:02.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.624382 25842 mvcc.cc:205] Tried to move back new op lower bound from 7333428767049658368 to 7333428766858739712. Current Snapshot: MvccSnapshot[applied={T|T < 7333428767049658368}]
01:58:02.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.636180 25839 sys_catalog.cc:455] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b4f1d98b13394b04986fe5948ab45ac8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } } }
01:58:02.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.637003 25839 sys_catalog.cc:458] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [sys.catalog]: This master's current role is: LEADER
01:58:02.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.640161 25841 catalog_manager.cc:1347] Generated new cluster ID: 84b07a1df93c423ab5754fdaa16ff70c
01:58:02.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.640436 25841 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:58:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.656144 25826 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b4f1d98b13394b04986fe5948ab45ac8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } } }
01:58:02.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.656929 25826 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483 [sys.catalog]: This master's current role is: FOLLOWER
01:58:02.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.671180 25841 catalog_manager.cc:1370] Generated new certificate authority record
01:58:02.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.673841 25841 catalog_manager.cc:1504] Loading token signing keys...
01:58:02.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.711481 25841 catalog_manager.cc:5868] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8: Generated new TSK 0
01:58:02.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.712991 25841 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.726792 25568 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/wal
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/data
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/logs
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/data/info.pb
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:58:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.248.1:0
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.248.1
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.248.1
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.248.62:42099,127.24.248.61:34905,127.24.248.60:36249
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.248.20:39725
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:02.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:58:02.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.802402 25723 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } attempt: 7
01:58:02.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.844223 25723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240: Bootstrap starting.
01:58:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.866653 25723 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240: Neither blocks nor log segments found. Creating new log.
01:58:02.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.873872 25723 log.cc:824] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240: Log is configured to *not* fsync() on all Append() calls
01:58:02.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.900699 25723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240: No bootstrap required, opened a new log
01:58:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.991430 25723 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } }
01:58:02.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.994486 25723 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:58:02.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.995739 25723 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3328fd2d10a1469e87124aa364669240, State: Initialized, Role: FOLLOWER
01:58:03.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:02.998595 25723 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } }
01:58:03.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.020817 25698 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b4f1d98b13394b04986fe5948ab45ac8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3328fd2d10a1469e87124aa364669240" is_pre_election: true
01:58:03.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.021760 25698 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b4f1d98b13394b04986fe5948ab45ac8 in term 0.
01:58:03.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.026104 25696 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [term 0 FOLLOWER]: Advancing to term 1
01:58:03.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.030015 25697 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b4f1d98b13394b04986fe5948ab45ac8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3328fd2d10a1469e87124aa364669240"
01:58:03.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.037103 25696 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [term 1 FOLLOWER]: Refusing update from remote peer b4f1d98b13394b04986fe5948ab45ac8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:58:03.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.036981 25850 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } } }
01:58:03.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.037829 25850 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [sys.catalog]: This master's current role is: FOLLOWER
01:58:03.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.039407 25723 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [sys.catalog]: configured and running, proceeding with master startup.
01:58:03.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.039354 25839 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:58:03.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.103186 25850 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b4f1d98b13394b04986fe5948ab45ac8. Latest consensus state: current_term: 1 leader_uuid: "b4f1d98b13394b04986fe5948ab45ac8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } } }
01:58:03.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.104104 25850 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [sys.catalog]: This master's current role is: FOLLOWER
01:58:03.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.104887 25856 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b4f1d98b13394b04986fe5948ab45ac8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } } }
01:58:03.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.105918 25856 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240 [sys.catalog]: This master's current role is: FOLLOWER
01:58:03.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.143780 25866 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
01:58:03.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:03.233131 25847 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:58:03.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:03.234124 25847 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:58:03.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:03.235134 25847 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:58:03.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:03.290851 25847 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:58:03.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:03.292564 25847 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.248.1
01:58:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.353293 25847 tablet_server_runner.cc:78] Tablet server non-default flags:
01:58:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.248.20:39725
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/data
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/wal
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.248.1:0
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/data/info.pb
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.248.1
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:03.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.248.62:42099,127.24.248.61:34905,127.24.248.60:36249
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.248.1
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/logs
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
01:58:03.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:58:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.354839 25847 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:58:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.356895 25847 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:58:03.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:03.379222 25872 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:03.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.431996 25837 catalog_manager.cc:1259] Loaded cluster ID: 84b07a1df93c423ab5754fdaa16ff70c
01:58:03.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.432346 25837 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483: loading cluster ID for follower catalog manager: success
01:58:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.455287 25837 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483: acquiring CA information for follower catalog manager: success
01:58:03.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.463337 25837 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ad1ef6621d884f90900c0d1ce63d4483: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:58:03.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.586848 25839 sys_catalog.cc:455] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b4f1d98b13394b04986fe5948ab45ac8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad1ef6621d884f90900c0d1ce63d4483" member_type: VOTER last_known_addr { host: "127.24.248.62" port: 42099 } } peers { permanent_uuid: "3328fd2d10a1469e87124aa364669240" member_type: VOTER last_known_addr { host: "127.24.248.61" port: 34905 } } peers { permanent_uuid: "b4f1d98b13394b04986fe5948ab45ac8" member_type: VOTER last_known_addr { host: "127.24.248.60" port: 36249 } } }
01:58:03.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:03.588356 25839 sys_catalog.cc:458] T 00000000000000000000000000000000 P b4f1d98b13394b04986fe5948ab45ac8 [sys.catalog]: This master's current role is: LEADER
01:58:04.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:04.155210 25866 catalog_manager.cc:1259] Loaded cluster ID: 84b07a1df93c423ab5754fdaa16ff70c
01:58:04.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:04.155587 25866 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240: loading cluster ID for follower catalog manager: success
01:58:04.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:04.192037 25866 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240: acquiring CA information for follower catalog manager: success
01:58:04.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:04.214156 25866 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 3328fd2d10a1469e87124aa364669240: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:58:06.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:03.380399 25873 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:06.606735 25847 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 3.228s user 0.943s sys 1.598s
01:58:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:04.773883 25871 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 25847
01:58:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:06.608386 25847 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 3.230s user 0.944s sys 1.599s
01:58:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:06.609445 25875 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:06.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:06.613551 25874 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:58:06.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.613584 25847 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:58:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.615003 25847 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:58:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.617568 25847 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:58:06.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.619001 25847 hybrid_clock.cc:648] HybridClock initialized: now 1790387886618858 us; error 134 us; skew 500 ppm
01:58:06.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.620042 25847 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:58:06.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.627418 25847 webserver.cc:460] Webserver started at http://127.24.248.1:34827/ using document root <none> and password file <none>
01:58:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.628561 25847 fs_manager.cc:359] Metadata directory not provided
01:58:06.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.628799 25847 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:58:06.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.629323 25847 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:58:06.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.634142 25847 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/data/instance:
01:58:06.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee3521c9d1e6406dbbc578658b3efa41"
01:58:06.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:06 on dist-test-slave-52vw"
01:58:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.635318 25847 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/wal/instance:
01:58:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee3521c9d1e6406dbbc578658b3efa41"
01:58:06.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:06 on dist-test-slave-52vw"
01:58:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.642799 25847 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.000s sys 0.006s
01:58:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.648281 25886 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:58:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.649592 25847 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.001s
01:58:06.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.649994 25847 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/data,/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/wal
01:58:06.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee3521c9d1e6406dbbc578658b3efa41"
01:58:06.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:06 on dist-test-slave-52vw"
01:58:06.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.650292 25847 fs_report.cc:383] FS layout report
01:58:06.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:58:06.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/wal
01:58:06.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/wal
01:58:06.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/data/data
01:58:06.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:58:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:58:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:58:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:58:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:58:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:58:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:58:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:58:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:58:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:58:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:58:06.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.683403 25847 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:58:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.685122 25847 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:58:06.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.685494 25847 kserver.cc:163] Server-wide thread pool size limit: 3276
01:58:06.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.687944 25847 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:58:06.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.692049 25847 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:58:06.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.692281 25847 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:58:06.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.692611 25847 ts_tablet_manager.cc:610] Registered 0 tablets
01:58:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.692822 25847 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:58:06.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.870219 25847 rpc_server.cc:307] RPC server started. Bound to: 127.24.248.1:43863
01:58:06.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.870409 25998 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.248.1:43863 every 8 connection(s)
01:58:06.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.872745 25847 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/data/info.pb
01:58:06.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.881219 25568 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu as pid 25847
01:58:06.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.881763 25568 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-0/wal/instance
01:58:06.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.916618 25999 heartbeater.cc:347] Connected to a master server at 127.24.248.60:36249
01:58:06.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.917115 25999 heartbeater.cc:464] Registering TS with master...
01:58:06.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.918234 25999 heartbeater.cc:511] Master 127.24.248.60:36249 requested a full tablet report, sending...
01:58:06.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.921129 25766 ts_manager.cc:194] Registered new tserver with Master: ee3521c9d1e6406dbbc578658b3efa41 (127.24.248.1:43863)
01:58:06.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.923987 25766 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.248.1:46617
01:58:06.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.926530 25568 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:06.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/wal
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/data
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/logs
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/data/info.pb
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:58:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.248.2:0
01:58:06.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.248.2
01:58:06.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.248.2
01:58:06.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:06.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.248.62:42099,127.24.248.61:34905,127.24.248.60:36249
01:58:06.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.248.20:39725
01:58:06.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:06.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:06.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:58:06.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.941875 26001 heartbeater.cc:347] Connected to a master server at 127.24.248.62:42099
01:58:06.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.942193 26001 heartbeater.cc:464] Registering TS with master...
01:58:06.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.942876 26001 heartbeater.cc:511] Master 127.24.248.62:42099 requested a full tablet report, sending...
01:58:06.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.945765 25613 ts_manager.cc:194] Registered new tserver with Master: ee3521c9d1e6406dbbc578658b3efa41 (127.24.248.1:43863)
01:58:06.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.950197 26000 heartbeater.cc:347] Connected to a master server at 127.24.248.61:34905
01:58:06.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.950516 26000 heartbeater.cc:464] Registering TS with master...
01:58:06.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.951188 26000 heartbeater.cc:511] Master 127.24.248.61:34905 requested a full tablet report, sending...
01:58:06.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:06.954489 25688 ts_manager.cc:194] Registered new tserver with Master: ee3521c9d1e6406dbbc578658b3efa41 (127.24.248.1:43863)
01:58:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:07.256458 26010 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:58:07.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:07.257119 26010 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:58:07.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:07.257915 26010 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:58:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:07.289534 26010 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:58:07.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:07.290494 26010 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.248.2
01:58:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:07.331995 26010 tablet_server_runner.cc:78] Tablet server non-default flags:
01:58:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.248.20:39725
01:58:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/data
01:58:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/wal
01:58:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.248.2:0
01:58:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:58:07.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:07.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:07.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/data/info.pb
01:58:07.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.248.2
01:58:07.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:07.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.248.62:42099,127.24.248.61:34905,127.24.248.60:36249
01:58:07.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:58:07.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:58:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:58:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.248.2
01:58:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/logs
01:58:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:58:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:58:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:58:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:58:07.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:58:07.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:58:07.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
01:58:07.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
01:58:07.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:58:07.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:07.333582 26010 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:58:07.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:07.335619 26010 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:58:07.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:07.356681 26018 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:07.930622 25999 heartbeater.cc:503] Master 127.24.248.60:36249 was elected leader, sending a full tablet report...
01:58:08.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:07.358652 26017 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:08.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:07.356768 26020 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:08.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:08.631887 26019 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1271 milliseconds
01:58:08.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.632012 26010 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:58:08.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.633289 26010 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:58:08.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.635412 26010 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:58:08.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.636770 26010 hybrid_clock.cc:648] HybridClock initialized: now 1790387888636753 us; error 50 us; skew 500 ppm
01:58:08.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.637773 26010 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:58:08.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.643999 26010 webserver.cc:460] Webserver started at http://127.24.248.2:45089/ using document root <none> and password file <none>
01:58:08.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.644893 26010 fs_manager.cc:359] Metadata directory not provided
01:58:08.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.645100 26010 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:58:08.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.645480 26010 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:58:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.650100 26010 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/data/instance:
01:58:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25b0da24e32b4cd3851ef7f057e46421"
01:58:08.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:08 on dist-test-slave-52vw"
01:58:08.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.651384 26010 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/wal/instance:
01:58:08.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25b0da24e32b4cd3851ef7f057e46421"
01:58:08.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:08 on dist-test-slave-52vw"
01:58:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.658175 26010 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.001s sys 0.008s
01:58:08.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.663686 26027 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:58:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.664937 26010 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
01:58:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.665197 26010 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/data,/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/wal
01:58:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "25b0da24e32b4cd3851ef7f057e46421"
01:58:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:08 on dist-test-slave-52vw"
01:58:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.665472 26010 fs_report.cc:383] FS layout report
01:58:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:58:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/wal
01:58:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/wal
01:58:08.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/data/data
01:58:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:58:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:58:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:58:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:58:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:58:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:58:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:58:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:58:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:58:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:58:08.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:58:08.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.707650 26010 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:58:08.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.709132 26010 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:58:08.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.709558 26010 kserver.cc:163] Server-wide thread pool size limit: 3276
01:58:08.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.711979 26010 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:58:08.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.716430 26010 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:58:08.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.716691 26010 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:58:08.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.716990 26010 ts_tablet_manager.cc:610] Registered 0 tablets
01:58:08.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.717195 26010 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:58:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.856688 26010 rpc_server.cc:307] RPC server started. Bound to: 127.24.248.2:46369
01:58:08.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.856777 26139 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.248.2:46369 every 8 connection(s)
01:58:08.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.859090 26010 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/data/info.pb
01:58:08.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.864511 25568 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu as pid 26010
01:58:08.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.865348 25568 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-1/wal/instance
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.898558 25568 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/wal
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/data
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/logs
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/data/info.pb
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.248.3:0
01:58:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.248.3
01:58:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.248.3
01:58:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.248.62:42099,127.24.248.61:34905,127.24.248.60:36249
01:58:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.248.20:39725
01:58:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:58:08.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.917049 26142 heartbeater.cc:347] Connected to a master server at 127.24.248.62:42099
01:58:08.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.917534 26142 heartbeater.cc:464] Registering TS with master...
01:58:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.918711 26142 heartbeater.cc:511] Master 127.24.248.62:42099 requested a full tablet report, sending...
01:58:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.920554 26141 heartbeater.cc:347] Connected to a master server at 127.24.248.61:34905
01:58:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.920924 26141 heartbeater.cc:464] Registering TS with master...
01:58:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.921849 26141 heartbeater.cc:511] Master 127.24.248.61:34905 requested a full tablet report, sending...
01:58:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.922192 25613 ts_manager.cc:194] Registered new tserver with Master: 25b0da24e32b4cd3851ef7f057e46421 (127.24.248.2:46369)
01:58:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.925230 26140 heartbeater.cc:347] Connected to a master server at 127.24.248.60:36249
01:58:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.925246 25688 ts_manager.cc:194] Registered new tserver with Master: 25b0da24e32b4cd3851ef7f057e46421 (127.24.248.2:46369)
01:58:08.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.925557 26140 heartbeater.cc:464] Registering TS with master...
01:58:08.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.926515 26140 heartbeater.cc:511] Master 127.24.248.60:36249 requested a full tablet report, sending...
01:58:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.928350 25766 ts_manager.cc:194] Registered new tserver with Master: 25b0da24e32b4cd3851ef7f057e46421 (127.24.248.2:46369)
01:58:08.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:08.930379 25766 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.248.2:50311
01:58:09.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:09.210521 26150 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:58:09.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:09.210981 26150 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:58:09.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:09.211390 26150 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:58:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:09.243225 26150 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:58:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:09.243994 26150 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.248.3
01:58:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:09.277343 26150 tablet_server_runner.cc:78] Tablet server non-default flags:
01:58:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.248.20:39725
01:58:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/data
01:58:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/wal
01:58:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.248.3:0
01:58:09.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/data/info.pb
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.248.3
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.248.62:42099,127.24.248.61:34905,127.24.248.60:36249
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.248.3
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/logs
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:58:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:09.278425 26150 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:58:09.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:09.280043 26150 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:58:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:09.297210 26157 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:09.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:09.933599 26140 heartbeater.cc:503] Master 127.24.248.60:36249 was elected leader, sending a full tablet report...
01:58:10.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:09.297638 26158 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:10.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:09.299934 26150 server_base.cc:1034] running on GCE node
01:58:10.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:09.297668 26160 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:10.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.491616 26150 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:58:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.493896 26150 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:58:10.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.495280 26150 hybrid_clock.cc:648] HybridClock initialized: now 1790387890495241 us; error 39 us; skew 500 ppm
01:58:10.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.496234 26150 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:58:10.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.508373 26150 webserver.cc:460] Webserver started at http://127.24.248.3:34899/ using document root <none> and password file <none>
01:58:10.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.509457 26150 fs_manager.cc:359] Metadata directory not provided
01:58:10.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.509724 26150 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:58:10.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.510232 26150 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:58:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.515055 26150 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/data/instance:
01:58:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c72470269bb347cfbc5d921bb841057e"
01:58:10.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:10 on dist-test-slave-52vw"
01:58:10.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.516235 26150 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/wal/instance:
01:58:10.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c72470269bb347cfbc5d921bb841057e"
01:58:10.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:10 on dist-test-slave-52vw"
01:58:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.523339 26150 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
01:58:10.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.528782 26167 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:58:10.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.530005 26150 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
01:58:10.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.530280 26150 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/data,/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/wal
01:58:10.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c72470269bb347cfbc5d921bb841057e"
01:58:10.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:10 on dist-test-slave-52vw"
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.530598 26150 fs_report.cc:383] FS layout report
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/wal
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/wal
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/data/data
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:58:10.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:58:10.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.565762 26150 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:58:10.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.567137 26150 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:58:10.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.567613 26150 kserver.cc:163] Server-wide thread pool size limit: 3276
01:58:10.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.570184 26150 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:58:10.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.574265 26150 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:58:10.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.574440 26150 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:58:10.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.574621 26150 ts_tablet_manager.cc:610] Registered 0 tablets
01:58:10.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.574749 26150 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:58:10.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.711560 26150 rpc_server.cc:307] RPC server started. Bound to: 127.24.248.3:37101
01:58:10.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.711637 26279 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.248.3:37101 every 8 connection(s)
01:58:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.714057 26150 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/data/info.pb
01:58:10.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.724414 25568 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu as pid 26150
01:58:10.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.725189 25568 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster7277501528114459605/ts-2/wal/instance
01:58:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.783478 26282 heartbeater.cc:347] Connected to a master server at 127.24.248.62:42099
01:58:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.785483 26282 heartbeater.cc:464] Registering TS with master...
01:58:10.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.786613 26282 heartbeater.cc:511] Master 127.24.248.62:42099 requested a full tablet report, sending...
01:58:10.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.787803 26280 heartbeater.cc:347] Connected to a master server at 127.24.248.60:36249
01:58:10.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.788100 26280 heartbeater.cc:464] Registering TS with master...
01:58:10.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.788862 26280 heartbeater.cc:511] Master 127.24.248.60:36249 requested a full tablet report, sending...
01:58:10.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.789354 25613 ts_manager.cc:194] Registered new tserver with Master: c72470269bb347cfbc5d921bb841057e (127.24.248.3:37101)
01:58:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.790745 26281 heartbeater.cc:347] Connected to a master server at 127.24.248.61:34905
01:58:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.790987 26281 heartbeater.cc:464] Registering TS with master...
01:58:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.791368 25766 ts_manager.cc:194] Registered new tserver with Master: c72470269bb347cfbc5d921bb841057e (127.24.248.3:37101)
01:58:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.791527 26281 heartbeater.cc:511] Master 127.24.248.61:34905 requested a full tablet report, sending...
01:58:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.793495 25766 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.248.3:34831
01:58:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.793622 25688 ts_manager.cc:194] Registered new tserver with Master: c72470269bb347cfbc5d921bb841057e (127.24.248.3:37101)
01:58:10.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:10.811041 25568 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
01:58:10.824 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:58:10.826 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
01:58:10.844 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.24.248.62"
port: 42099
}
bound_http_address {
host: "127.24.248.62"
port: 46357
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.24.248.61"
port: 34905
}
bound_http_address {
host: "127.24.248.61"
port: 45183
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.24.248.60"
port: 36249
}
bound_http_address {
host: "127.24.248.60"
port: 32889
}
}
}
01:58:10.849 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:58:10.852 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.24.248.1"
port: 43863
}
bound_http_address {
host: "127.24.248.1"
port: 34827
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.24.248.2"
port: 46369
}
bound_http_address {
host: "127.24.248.2"
port: 45089
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.24.248.3"
port: 37101
}
bound_http_address {
host: "127.24.248.3"
port: 34899
}
}
}
01:58:10.852 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
01:58:11.027 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
01:58:11.141 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskaDlrHb/test-tmp
01:58:11.142 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:58:11.142 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskaDlrHb/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:58:11.146 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
01:58:11.517 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:58:11.518 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:58:11.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:11.520700 26292 mini_oidc.cc:152] Starting JWKS server
01:58:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:11.524000 26292 webserver.cc:460] Webserver started at http://127.0.0.1:43003/ using document root <none> and password file <none>
01:58:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:11.524364 26292 mini_oidc.cc:189] Starting OIDC Discovery server
01:58:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:11.526046 26292 webserver.cc:460] Webserver started at http://127.0.0.1:34783/ using document root <none> and password file <none>
01:58:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
01:58:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/principal' for realm 'KRBTEST.COM',
01:58:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
01:58:11.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:11 dist-test-slave-52vw krb5kdc[26304](info): setting up network...
01:58:11.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
01:58:11.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:11 dist-test-slave-52vw krb5kdc[26304](info): set up 2 sockets
01:58:11.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:11 dist-test-slave-52vw krb5kdc[26304](info): commencing operation
01:58:11.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
01:58:11.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:11.797883 26280 heartbeater.cc:503] Master 127.24.248.60:36249 was elected leader, sending a full tablet report...
01:58:12.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:12.623466 26292 mini_kdc.cc:121] Time spent starting KDC: real 1.072s user 0.001s sys 0.016s
01:58:12.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
01:58:12.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:58:12.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
01:58:12.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
01:58:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:58:12.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
01:58:12.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
01:58:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:58:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
01:58:12.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:58:12.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/test-user.keytab.
01:58:12.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/test-user.keytab.
01:58:12.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:12 dist-test-slave-52vw krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387892, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:58:12.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:58:12.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T01:58:12Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:58:12.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T01:58:12Z Disabled control of system clock
01:58:12.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.173.62@KRBTEST.COM; defaulting to no policy
01:58:12.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:58:12.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.173.62@KRBTEST.COM" created.
01:58:12.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:58:12.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.62.keytab.
01:58:12.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.62.keytab.
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:12.817987 26292 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/logs
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data/info.pb
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.62:42167
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.62
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.62.keytab
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.62
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:43057
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43003/jwks.json
01:58:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:58:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:13.120493 26317 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:58:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:13.121199 26317 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:58:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:13.122103 26317 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:58:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:13.153578 26317 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:58:13.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:13.153863 26317 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:58:13.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:13.154245 26317 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:58:13.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:13.154567 26317 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:58:13.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:13.154867 26317 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:58:13.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:13.155179 26317 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:58:13.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:13.155409 26317 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43003/jwks.json
01:58:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:13.194955 26317 master_runner.cc:386] Master server non-default flags:
01:58:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:43057
01:58:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data
01:58:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal
01:58:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:58:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419
01:58:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.62.keytab
01:58:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.62
01:58:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.62:42167
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43003/jwks.json
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data/info.pb
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.62
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/logs
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:58:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:58:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
01:58:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
01:58:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:58:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:13.196537 26317 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:58:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:13.197718 26317 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:58:13.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:13.218545 26327 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:14.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:13.222621 26324 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:14.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:13.219650 26325 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:14.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:14.439384 26326 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:58:14.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.439507 26317 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:58:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.443506 26317 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:58:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.446213 26317 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:58:14.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.447609 26317 hybrid_clock.cc:648] HybridClock initialized: now 1790387894447560 us; error 62 us; skew 500 ppm
01:58:14.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.448599 26317 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:58:14.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:14 dist-test-slave-52vw krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387894, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:58:14.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.454414 26317 init.cc:377] Logged in from keytab as kudu/127.25.173.62@KRBTEST.COM (short username kudu)
01:58:14.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.459435 26317 webserver.cc:460] Webserver started at http://127.25.173.62:41697/ using document root <none> and password file <none>
01:58:14.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.460381 26317 fs_manager.cc:359] Metadata directory not provided
01:58:14.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.460548 26317 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:58:14.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.460983 26317 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:58:14.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.465662 26317 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data/instance:
01:58:14.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f0776e84fabb434d8af2316722bc7766"
01:58:14.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:14 on dist-test-slave-52vw"
01:58:14.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.466670 26317 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal/instance:
01:58:14.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f0776e84fabb434d8af2316722bc7766"
01:58:14.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:14 on dist-test-slave-52vw"
01:58:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.473634 26317 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
01:58:14.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.478644 26335 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:58:14.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.479566 26317 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
01:58:14.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.479887 26317 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data,/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal
01:58:14.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f0776e84fabb434d8af2316722bc7766"
01:58:14.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:14 on dist-test-slave-52vw"
01:58:14.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.480265 26317 fs_report.cc:383] FS layout report
01:58:14.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:58:14.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal
01:58:14.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal
01:58:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data/data
01:58:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:58:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:58:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:58:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:58:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:58:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:58:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:58:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:58:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:58:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:58:14.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:58:14.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.538932 26317 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:58:14.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.544844 26317 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:58:14.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.545353 26317 kserver.cc:163] Server-wide thread pool size limit: 3276
01:58:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.615624 26317 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.62:42167
01:58:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.615679 26386 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.173.62:42167 every 8 connection(s)
01:58:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.618377 26317 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data/info.pb
01:58:14.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.623241 26387 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:58:14.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.627789 26292 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu as pid 26317
01:58:14.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.628183 26292 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal/instance
01:58:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.634626 26387 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } has no permanent_uuid. Determining permanent_uuid...
01:58:14.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.173.61@KRBTEST.COM; defaulting to no policy
01:58:14.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:58:14.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.173.61@KRBTEST.COM" created.
01:58:14.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:14 dist-test-slave-52vw krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387894, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
01:58:14.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:58:14.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.61.keytab.
01:58:14.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.61.keytab.
01:58:14.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.673383 26292 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:14.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:14.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/logs
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data/info.pb
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.61:43197
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.61
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.61.keytab
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.61
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:43057
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43003/jwks.json
01:58:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:58:14.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.673674 26387 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } has no permanent_uuid. Determining permanent_uuid...
01:58:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:14.676082 26338 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:14.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:14.680243 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.61:43197: Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:14.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.729779 26387 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } attempt: 1
01:58:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:14.734087 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.61:43197: Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:14.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.802492 26387 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } attempt: 2
01:58:14.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:14.805923 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.61:43197: Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:14.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:14.897372 26387 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } attempt: 3
01:58:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:14.900990 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.61:43197: Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:14.991938 26394 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:58:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:14.992475 26394 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:58:14.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:14.993301 26394 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:58:15.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:15.024713 26394 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:58:15.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:15.025018 26394 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:58:15.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:15.025404 26394 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:58:15.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:15.025792 26394 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:58:15.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:15.026094 26394 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:58:15.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:15.026399 26394 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:58:15.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:15.026629 26394 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43003/jwks.json
01:58:15.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:15.044430 26387 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } attempt: 4
01:58:15.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:15.047940 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.61:43197: Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:15.063925 26394 master_runner.cc:386] Master server non-default flags:
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:43057
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.61.keytab
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.61
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.61:43197
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43003/jwks.json
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data/info.pb
01:58:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.61
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/logs
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:58:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:15.065127 26394 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:58:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:15.066180 26394 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:58:15.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:15.084394 26400 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:15.347388 26387 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } attempt: 5
01:58:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:15.351779 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.61:43197: Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:15.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:15.899343 26387 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } attempt: 6
01:58:15.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:15.905699 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.61:43197: Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:16.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:15.084461 26401 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:16.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:15.086365 26394 server_base.cc:1034] running on GCE node
01:58:16.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:15.084522 26403 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:16.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.324035 26394 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:58:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.327284 26394 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:58:16.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.328715 26394 hybrid_clock.cc:648] HybridClock initialized: now 1790387896328678 us; error 58 us; skew 500 ppm
01:58:16.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.329762 26394 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:58:16.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:16 dist-test-slave-52vw krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387896, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:58:16.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.341198 26394 init.cc:377] Logged in from keytab as kudu/127.25.173.61@KRBTEST.COM (short username kudu)
01:58:16.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.346961 26394 webserver.cc:460] Webserver started at http://127.25.173.61:40919/ using document root <none> and password file <none>
01:58:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.347838 26394 fs_manager.cc:359] Metadata directory not provided
01:58:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.348019 26394 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:58:16.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.348457 26394 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:58:16.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.353266 26394 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data/instance:
01:58:16.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "941f53d6fb20487381a58726d84a3496"
01:58:16.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:16 on dist-test-slave-52vw"
01:58:16.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.354322 26394 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal/instance:
01:58:16.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "941f53d6fb20487381a58726d84a3496"
01:58:16.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:16 on dist-test-slave-52vw"
01:58:16.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.362061 26394 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
01:58:16.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.367609 26413 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:58:16.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.368762 26394 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
01:58:16.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.369213 26394 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data,/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal
01:58:16.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "941f53d6fb20487381a58726d84a3496"
01:58:16.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:16 on dist-test-slave-52vw"
01:58:16.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.369652 26394 fs_report.cc:383] FS layout report
01:58:16.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:58:16.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal
01:58:16.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal
01:58:16.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data/data
01:58:16.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:58:16.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:58:16.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:58:16.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:58:16.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:58:16.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:58:16.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:58:16.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:58:16.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:58:16.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:58:16.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:58:16.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.434008 26394 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:58:16.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.439829 26394 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:58:16.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.440347 26394 kserver.cc:163] Server-wide thread pool size limit: 3276
01:58:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.513970 26394 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.61:43197
01:58:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.514014 26464 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.173.61:43197 every 8 connection(s)
01:58:16.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.516650 26394 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data/info.pb
01:58:16.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.518432 26292 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu as pid 26394
01:58:16.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.518913 26292 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal/instance
01:58:16.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.522847 26465 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:58:16.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.534212 26465 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } has no permanent_uuid. Determining permanent_uuid...
01:58:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.173.60@KRBTEST.COM; defaulting to no policy
01:58:16.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:58:16.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.173.60@KRBTEST.COM" created.
01:58:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:16 dist-test-slave-52vw krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387896, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.61@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
01:58:16.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:58:16.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.60.keytab.
01:58:16.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.173.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.60.keytab.
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.570896 26292 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/wal
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/logs
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data/info.pb
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:58:16.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.60:33419
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.60
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.60.keytab
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.60
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:58:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:43057
01:58:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419
01:58:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43003/jwks.json
01:58:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:58:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.571375 26465 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } has no permanent_uuid. Determining permanent_uuid...
01:58:16.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:16 dist-test-slave-52vw krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387896, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.61@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
01:58:16.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.600644 26465 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } has no permanent_uuid. Determining permanent_uuid...
01:58:16.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.602806 26416 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:16.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.606627 26465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:16.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.656136 26465 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } attempt: 1
01:58:16.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.659875 26465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.728319 26465 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } attempt: 2
01:58:16.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.731957 26465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:16.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.823364 26465 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } attempt: 3
01:58:16.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.827133 26465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.888816 26472 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:58:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.889496 26472 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:58:16.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.890090 26472 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:58:16.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.920550 26472 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:58:16.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.920764 26472 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:58:16.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.921104 26472 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:58:16.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.921307 26472 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:58:16.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.921491 26472 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:58:16.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.921676 26472 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:58:16.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.921809 26472 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43003/jwks.json
01:58:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.960151 26472 master_runner.cc:386] Master server non-default flags:
01:58:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:43057
01:58:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data
01:58:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/wal
01:58:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:58:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419
01:58:16.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.60.keytab
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.60
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.60:33419
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43003/jwks.json
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data/info.pb
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.60
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/logs
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
01:58:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:58:16.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.961460 26472 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:58:16.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.962376 26472 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:58:16.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.966279 26387 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } attempt: 7
01:58:16.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.970614 26465 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } attempt: 4
01:58:16.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.975450 26465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:16.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:16 dist-test-slave-52vw krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387894, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
01:58:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.985004 26481 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:16.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:16.996286 26387 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } has no permanent_uuid. Determining permanent_uuid...
01:58:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:17.001395 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:17.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:17.059808 26387 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } attempt: 1
01:58:17.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:17.065775 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:17.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:17.147359 26387 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } attempt: 2
01:58:17.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:17.153469 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:17.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:17.239195 26387 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } attempt: 3
01:58:17.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:17.246969 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:17.274894 26465 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } attempt: 5
01:58:17.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:17.280632 26465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:17.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:17.387501 26387 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } attempt: 4
01:58:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:17.393143 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:17.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:17.676749 26387 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } attempt: 5
01:58:17.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:17.682577 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:17.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:17.828194 26465 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } attempt: 6
01:58:17.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:17.837566 26465 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.235168 26387 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } attempt: 6
01:58:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:18.250164 26387 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.173.60:33419: Network error: Client connection negotiation failed: client connection to 127.25.173.60:33419: connect: Connection refused (error 111)
01:58:18.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.988029 26482 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:18.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:16.985653 26484 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:18.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:18.317566 26483 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1331 milliseconds
01:58:18.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.317673 26472 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:58:18.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.318786 26472 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:58:18.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.324998 26472 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:58:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.326443 26472 hybrid_clock.cc:648] HybridClock initialized: now 1790387898326383 us; error 67 us; skew 500 ppm
01:58:18.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.327216 26472 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:58:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:18 dist-test-slave-52vw krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387898, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:58:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.332832 26472 init.cc:377] Logged in from keytab as kudu/127.25.173.60@KRBTEST.COM (short username kudu)
01:58:18.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.338910 26472 webserver.cc:460] Webserver started at http://127.25.173.60:44493/ using document root <none> and password file <none>
01:58:18.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.339775 26472 fs_manager.cc:359] Metadata directory not provided
01:58:18.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.339960 26472 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:58:18.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.340411 26472 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:58:18.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.345148 26472 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data/instance:
01:58:18.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab1bf874594a456993831d9a1e862f26"
01:58:18.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:18 on dist-test-slave-52vw"
01:58:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.346158 26472 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/wal/instance:
01:58:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab1bf874594a456993831d9a1e862f26"
01:58:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:18 on dist-test-slave-52vw"
01:58:18.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.353320 26472 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
01:58:18.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.358549 26499 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:58:18.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.359553 26472 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
01:58:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.359844 26472 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data,/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/wal
01:58:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab1bf874594a456993831d9a1e862f26"
01:58:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:18 on dist-test-slave-52vw"
01:58:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.360184 26472 fs_report.cc:383] FS layout report
01:58:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:58:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/wal
01:58:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/wal
01:58:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data/data
01:58:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:58:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:58:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:58:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:58:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:58:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:58:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:58:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:58:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:58:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:58:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:58:18.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.406983 26472 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:58:18.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.412444 26472 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:58:18.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.412923 26472 kserver.cc:163] Server-wide thread pool size limit: 3276
01:58:18.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.487699 26472 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.60:33419
01:58:18.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.487835 26550 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.173.60:33419 every 8 connection(s)
01:58:18.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.491201 26472 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data/info.pb
01:58:18.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.497013 26551 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:58:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.499402 26292 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu as pid 26472
01:58:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.499735 26292 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/wal/instance
01:58:18.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.510541 26551 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } has no permanent_uuid. Determining permanent_uuid...
01:58:18.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:18 dist-test-slave-52vw krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387898, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
01:58:18.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:18 dist-test-slave-52vw krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387892, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.173.60@KRBTEST.COM
01:58:18.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.548771 26551 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } has no permanent_uuid. Determining permanent_uuid...
01:58:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:18 dist-test-slave-52vw krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387898, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
01:58:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:18 dist-test-slave-52vw krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387892, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.173.61@KRBTEST.COM
01:58:18.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.574868 26551 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } has no permanent_uuid. Determining permanent_uuid...
01:58:18.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:18 dist-test-slave-52vw krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387898, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for kudu/127.25.173.60@KRBTEST.COM
01:58:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:18 dist-test-slave-52vw krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387892, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.173.62@KRBTEST.COM
01:58:18.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.605486 26551 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: Bootstrap starting.
01:58:18.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.606006 26292 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
01:58:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.612684 26551 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: Neither blocks nor log segments found. Creating new log.
01:58:18.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.615100 26551 log.cc:824] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: Log is configured to *not* fsync() on all Append() calls
01:58:18.616 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:58:18.618 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
01:58:18.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.620296 26551 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: No bootstrap required, opened a new log
01:58:18.631 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
01:58:18.632 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
01:58:18.635 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.25.173.62"
port: 42167
}
bound_http_address {
host: "127.25.173.62"
port: 41697
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.25.173.61"
port: 43197
}
bound_http_address {
host: "127.25.173.61"
port: 40919
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.25.173.60"
port: 33419
}
bound_http_address {
host: "127.25.173.60"
port: 44493
}
}
}
01:58:18.636 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:58:18.639 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:58:18.639 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:58:18.641 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:58:18.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.647884 26551 raft_consensus.cc:348] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } } peers { permanent_uuid: "941f53d6fb20487381a58726d84a3496" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } } peers { permanent_uuid: "ab1bf874594a456993831d9a1e862f26" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } }
01:58:18.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.648536 26551 raft_consensus.cc:374] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:58:18.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.649099 26551 raft_consensus.cc:729] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ab1bf874594a456993831d9a1e862f26, State: Initialized, Role: FOLLOWER
01:58:18.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.650151 26551 consensus_queue.cc:260] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } } peers { permanent_uuid: "941f53d6fb20487381a58726d84a3496" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } } peers { permanent_uuid: "ab1bf874594a456993831d9a1e862f26" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } }
01:58:18.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.653132 26561 sys_catalog.cc:455] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } } peers { permanent_uuid: "941f53d6fb20487381a58726d84a3496" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } } peers { permanent_uuid: "ab1bf874594a456993831d9a1e862f26" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } } }
01:58:18.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.653963 26561 sys_catalog.cc:458] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [sys.catalog]: This master's current role is: FOLLOWER
01:58:18.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.654821 26551 sys_catalog.cc:564] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [sys.catalog]: configured and running, proceeding with master startup.
01:58:18.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:18 dist-test-slave-52vw krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387898, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:58:18.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:58:18.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:18.667935 26505 tablet.cc:2369] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:58:18.675 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:58:18.678 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5cc
01:58:18.686 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:58:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:18.687317 26573 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:18.687 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5cc
01:58:18.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:18.687675 26573 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:18.689 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:58:18.693 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
01:58:18.694 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:58:18.695 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:58:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.679023 26472 master_runner.cc:186] Error getting master registration for 127.25.173.62:42167: OK, instance_id { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" instance_seqno: 1790387894594687 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:58:18.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.715457 26472 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:58:18.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:18 dist-test-slave-52vw krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387898, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:58:18.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:58:18.743 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:58:18.743 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5cc
01:58:18.744 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:58:18.745 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.173.62:42167
01:58:18.746 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
01:58:18.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.747254 26292 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu with pid 26317
01:58:18.773 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:58:18.773 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.173.61:43197
01:58:18.774 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
01:58:18.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.775297 26292 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu with pid 26394
01:58:18.796 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:58:18.797 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.173.60:33419
01:58:18.797 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
01:58:18.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.797725 26292 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu with pid 26472
01:58:18.820 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:58:18.820 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.173.62:42167
01:58:18.822 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:18.823799 26292 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/logs
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data/info.pb
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:58:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.62:42167
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.62
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41697
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.62.keytab
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.62
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:43057
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43003/jwks.json
01:58:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:58:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:19.136160 26577 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:58:19.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:19.136775 26577 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:58:19.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:19.137426 26577 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:58:19.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:19.168210 26577 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:58:19.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:19.168527 26577 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:58:19.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:19.168799 26577 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:58:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:19.169021 26577 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:58:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:19.169209 26577 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:58:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:19.169405 26577 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:58:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:19.169584 26577 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43003/jwks.json
01:58:19.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:19.210445 26577 master_runner.cc:386] Master server non-default flags:
01:58:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:43057
01:58:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data
01:58:19.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.62.keytab
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.62
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.62:42167
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43003/jwks.json
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data/info.pb
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.62
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41697
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/logs
01:58:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:58:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:58:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:58:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:58:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:58:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:58:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
01:58:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
01:58:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:58:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:19.211583 26577 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:58:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:19.212771 26577 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:58:19.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:19.232367 26588 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:20.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:19.233129 26586 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:20.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:19.233703 26585 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:20.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:19.233460 26577 server_base.cc:1034] running on GCE node
01:58:20.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.447818 26577 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:58:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.450963 26577 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:58:20.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.452422 26577 hybrid_clock.cc:648] HybridClock initialized: now 1790387900452384 us; error 50 us; skew 500 ppm
01:58:20.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.453522 26577 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:58:20.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:20 dist-test-slave-52vw krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387900, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:58:20.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.460454 26577 init.cc:377] Logged in from keytab as kudu/127.25.173.62@KRBTEST.COM (short username kudu)
01:58:20.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.466079 26577 webserver.cc:460] Webserver started at http://127.25.173.62:41697/ using document root <none> and password file <none>
01:58:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.467212 26577 fs_manager.cc:359] Metadata directory not provided
01:58:20.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.467471 26577 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:58:20.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.477640 26577 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.001s
01:58:20.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.482739 26596 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:58:20.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.483781 26577 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
01:58:20.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.484169 26577 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data,/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal
01:58:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f0776e84fabb434d8af2316722bc7766"
01:58:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:14 on dist-test-slave-52vw"
01:58:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.486987 26577 fs_report.cc:383] FS layout report
01:58:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:58:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal
01:58:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal
01:58:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data/data
01:58:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:58:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:58:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:58:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:58:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:58:20.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:58:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:58:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:58:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:58:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:58:20.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:58:20.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.524012 26577 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:58:20.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.529665 26577 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:58:20.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.530194 26577 kserver.cc:163] Server-wide thread pool size limit: 3276
01:58:20.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.602464 26577 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.62:42167
01:58:20.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.602587 26647 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.173.62:42167 every 8 connection(s)
01:58:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.605305 26577 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/data/info.pb
01:58:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.606699 26292 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu as pid 26577
01:58:20.607 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:58:20.607 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.173.61:43197
01:58:20.608 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.609871 26292 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/logs
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data/info.pb
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:58:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.61:43197
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.61
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40919
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.61.keytab
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.61
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:58:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:43057
01:58:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419
01:58:20.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:20.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43003/jwks.json
01:58:20.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:58:20.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.617630 26648 sys_catalog.cc:263] Verifying existing consensus state
01:58:20.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 01:58:20.618410 26648 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:58:20.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.618856 26577 master.cc:537] Master@127.25.173.62:42167 shutting down...
01:58:20.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:20.678109 26577 master.cc:552] Master@127.25.173.62:42167 shutdown complete.
01:58:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:58:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:20.951272 26650 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:58:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:20.951797 26650 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:58:20.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:20.952335 26650 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:58:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:20.985682 26650 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:58:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:20.985898 26650 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:58:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:20.986168 26650 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:58:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:20.986356 26650 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:58:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:20.986591 26650 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:58:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:20.986878 26650 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:58:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:20.987114 26650 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43003/jwks.json
01:58:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:21.025202 26650 master_runner.cc:386] Master server non-default flags:
01:58:21.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:43057
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.61.keytab
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.61
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.61:43197
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43003/jwks.json
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data/info.pb
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.61
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40919
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:58:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:58:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/logs
01:58:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:58:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:58:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:58:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:58:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:58:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:58:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
01:58:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
01:58:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:58:21.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:21.026667 26650 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:58:21.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:21.027977 26650 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:58:21.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:21.045593 26657 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:22.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:21.045615 26660 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:22.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:21.046240 26658 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:22.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.270684 26659 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:58:22.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.270802 26650 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:58:22.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.274878 26650 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:58:22.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.277410 26650 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:58:22.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.278772 26650 hybrid_clock.cc:648] HybridClock initialized: now 1790387902278745 us; error 35 us; skew 500 ppm
01:58:22.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.279541 26650 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:58:22.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:22 dist-test-slave-52vw krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387902, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:58:22.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.291412 26650 init.cc:377] Logged in from keytab as kudu/127.25.173.61@KRBTEST.COM (short username kudu)
01:58:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.296736 26650 webserver.cc:460] Webserver started at http://127.25.173.61:40919/ using document root <none> and password file <none>
01:58:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.297865 26650 fs_manager.cc:359] Metadata directory not provided
01:58:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.298177 26650 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:58:22.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.305673 26650 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.004s sys 0.000s
01:58:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.309753 26669 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:58:22.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.310681 26650 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
01:58:22.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.311040 26650 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data,/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal
01:58:22.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "941f53d6fb20487381a58726d84a3496"
01:58:22.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:16 on dist-test-slave-52vw"
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.313463 26650 fs_report.cc:383] FS layout report
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data/data
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:58:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:58:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.350836 26650 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:58:22.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.356271 26650 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:58:22.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.356774 26650 kserver.cc:163] Server-wide thread pool size limit: 3276
01:58:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.425992 26650 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.61:43197
01:58:22.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.426110 26720 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.173.61:43197 every 8 connection(s)
01:58:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.428714 26650 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/data/info.pb
01:58:22.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.433703 26292 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu as pid 26650
01:58:22.435 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:58:22.435 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.173.60:33419
01:58:22.435 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.436728 26292 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/wal
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/logs
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data/info.pb
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:22.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.60:33419
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.60
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44493
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.60.keytab
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.60
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:43057
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43003/jwks.json
01:58:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:58:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.440804 26721 sys_catalog.cc:263] Verifying existing consensus state
01:58:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 01:58:22.441659 26721 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:58:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.442044 26650 master.cc:537] Master@127.25.173.61:43197 shutting down...
01:58:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.503492 26650 master.cc:552] Master@127.25.173.61:43197 shutdown complete.
01:58:22.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:58:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.764266 26723 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:58:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.764959 26723 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:58:22.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.765805 26723 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:58:22.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.797089 26723 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:58:22.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.797361 26723 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:58:22.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.797737 26723 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:58:22.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.798058 26723 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:58:22.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.798364 26723 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:58:22.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.798669 26723 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:58:22.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.798889 26723 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43003/jwks.json
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.834815 26723 master_runner.cc:386] Master server non-default flags:
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.173.20:43057
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/wal
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/krb5kdc/kudu_127.25.173.60.keytab
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.173.60
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.173.60:33419
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:58:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43003/jwks.json
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data/info.pb
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.173.60
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44493
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/logs
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 00:58:27 UTC on ccycloud.quasar-hahlai.root.comops.site
01:58:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673502
01:58:22.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:58:22.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.836287 26723 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:58:22.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.837584 26723 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:58:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.857359 26730 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:24.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.858187 26729 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:24.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:22.860355 26723 server_base.cc:1034] running on GCE node
01:58:24.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:22.859468 26732 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:58:24.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.120218 26723 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:58:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.122754 26723 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:58:24.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.124155 26723 hybrid_clock.cc:648] HybridClock initialized: now 1790387904124124 us; error 47 us; skew 500 ppm
01:58:24.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.125113 26723 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:58:24.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:24 dist-test-slave-52vw krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387904, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:58:24.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.130897 26723 init.cc:377] Logged in from keytab as kudu/127.25.173.60@KRBTEST.COM (short username kudu)
01:58:24.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.135962 26723 webserver.cc:460] Webserver started at http://127.25.173.60:44493/ using document root <none> and password file <none>
01:58:24.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.137017 26723 fs_manager.cc:359] Metadata directory not provided
01:58:24.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.137261 26723 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:58:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.145087 26723 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.002s sys 0.005s
01:58:24.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.149824 26741 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:58:24.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.150975 26723 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
01:58:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.151345 26723 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data,/tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/wal
01:58:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab1bf874594a456993831d9a1e862f26"
01:58:24.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 01:58:18 on dist-test-slave-52vw"
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.153705 26723 fs_report.cc:383] FS layout report
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/wal
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/wal
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data/data
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:58:24.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:58:24.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.202586 26723 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:58:24.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.208603 26723 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:58:24.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.209164 26723 kserver.cc:163] Server-wide thread pool size limit: 3276
01:58:24.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.284168 26723 rpc_server.cc:307] RPC server started. Bound to: 127.25.173.60:33419
01:58:24.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.284286 26792 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.173.60:33419 every 8 connection(s)
01:58:24.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.287061 26723 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaDlrHb/test-tmp/mini-kudu-cluster2419576363470067501/master-2/data/info.pb
01:58:24.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.292021 26292 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu as pid 26723
01:58:24.292 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:58:24.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.298426 26793 sys_catalog.cc:263] Verifying existing consensus state
01:58:24.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.304481 26793 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: Bootstrap starting.
01:58:24.338 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:24.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.346024 26793 log.cc:824] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: Log is configured to *not* fsync() on all Append() calls
01:58:24.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.351279 26793 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:58:24.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.351925 26793 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: Bootstrap complete.
01:58:24.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.381168 26793 raft_consensus.cc:348] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } } peers { permanent_uuid: "941f53d6fb20487381a58726d84a3496" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } } peers { permanent_uuid: "ab1bf874594a456993831d9a1e862f26" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } }
01:58:24.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.384613 26793 raft_consensus.cc:374] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:58:24.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.384943 26793 raft_consensus.cc:729] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ab1bf874594a456993831d9a1e862f26, State: Initialized, Role: FOLLOWER
01:58:24.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.385991 26793 consensus_queue.cc:260] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } } peers { permanent_uuid: "941f53d6fb20487381a58726d84a3496" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } } peers { permanent_uuid: "ab1bf874594a456993831d9a1e862f26" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } }
01:58:24.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.389009 26797 sys_catalog.cc:455] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } } peers { permanent_uuid: "941f53d6fb20487381a58726d84a3496" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } } peers { permanent_uuid: "ab1bf874594a456993831d9a1e862f26" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } } }
01:58:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.390067 26797 sys_catalog.cc:458] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [sys.catalog]: This master's current role is: FOLLOWER
01:58:24.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.390781 26793 sys_catalog.cc:564] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [sys.catalog]: configured and running, proceeding with master startup.
01:58:24.405 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:24.422845 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:24.424019 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.433445 26797 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:58:24.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.433974 26797 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } } peers { permanent_uuid: "941f53d6fb20487381a58726d84a3496" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } } peers { permanent_uuid: "ab1bf874594a456993831d9a1e862f26" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } }
01:58:24.435 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.173.60:33419(127.25.173.60:33419)] connecting to peer
01:58:24.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.437444 26797 leader_election.cc:288] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167), 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197)
01:58:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:24.441684 26744 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111)
01:58:24.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:24.454311 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167): Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111)
01:58:24.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:24.455236 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197): Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:24.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.456468 26744 leader_election.cc:302] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab1bf874594a456993831d9a1e862f26; no voters: 941f53d6fb20487381a58726d84a3496, f0776e84fabb434d8af2316722bc7766
01:58:24.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.457432 26797 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:58:24.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.426100 26723 master_runner.cc:186] Error getting master registration for 127.25.173.62:42167: Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111),
01:58:24.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:24.463019 26723 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:58:24.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:24.464694 26747 tablet.cc:2369] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:58:24.497 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:24.497 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:24.497 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:24.498 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:24.501 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:24.501 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:24.503 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.25.173.60:33419(127.25.173.60:33419)] Successfully connected to peer
01:58:24.588 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
01:58:24.597 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.25.173.60
01:58:24.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:24 dist-test-slave-52vw krb5kdc[26304](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387898, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.173.60@KRBTEST.COM
01:58:24.886 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x8f08b04f, L:/127.0.0.1:34332 - R:/127.25.173.60:33419] using SASL/GSSAPI
01:58:24.908 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:24.938 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:24.940 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:24.945 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:24.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:24.946 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:24.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:24.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:24.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:24.953 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:24.978 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:24.978 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:24.981 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:24.981 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:24.982 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:24.984 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:24.984 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:24.985 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:24.986 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:24.998 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:24.999 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:25.000 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.001 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.001 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.002 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.002 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.003 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.005 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:25.018 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:25.019 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:25.023 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.024 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.026 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.026 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.027 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.029 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:25.077 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:25.079 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:25.080 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.080 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.081 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.083 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.083 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.083 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.085 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:25.117 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:25.119 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:25.120 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.120 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.121 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.124 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.124 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.126 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:25.218 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:25.222 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.222 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.223 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.225 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:25.227 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.227 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.227 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:25.418 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:25.420 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.420 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.421 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:25.422 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.423 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.425 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.425 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:25.428261 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:25.429 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:25.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:25.428610 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:25.637 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:25.638 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:25.641 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.642 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.642 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.642 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.643 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:25.643 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:25.644 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:25.961793 26816 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:58:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:25.962220 26816 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } } peers { permanent_uuid: "941f53d6fb20487381a58726d84a3496" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } } peers { permanent_uuid: "ab1bf874594a456993831d9a1e862f26" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } }
01:58:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:25.963567 26816 leader_election.cc:288] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167), 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197)
01:58:25.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:25.970386 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167): Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111)
01:58:25.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:25.971113 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197): Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:25.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:25.971401 26744 leader_election.cc:302] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab1bf874594a456993831d9a1e862f26; no voters: 941f53d6fb20487381a58726d84a3496, f0776e84fabb434d8af2316722bc7766
01:58:25.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:25.972478 26816 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:58:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:26.431581 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:26.431866 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:26.537 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:26.538 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:26.539 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:26.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:26.540 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:26.541 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:26.541 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:26.541 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:26.543 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:27.434715 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:27.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:27.435022 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:27.861025 26823 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:58:27.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:27.861493 26823 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } } peers { permanent_uuid: "941f53d6fb20487381a58726d84a3496" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } } peers { permanent_uuid: "ab1bf874594a456993831d9a1e862f26" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } }
01:58:27.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:27.863011 26823 leader_election.cc:288] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167), 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197)
01:58:27.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:27.868686 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167): Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111)
01:58:27.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:27.869493 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197): Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:27.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:27.869869 26744 leader_election.cc:302] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab1bf874594a456993831d9a1e862f26; no voters: 941f53d6fb20487381a58726d84a3496, f0776e84fabb434d8af2316722bc7766
01:58:27.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:27.870507 26823 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:58:28.137 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:28.138 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:28.139 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:28.140 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:28.140 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:28.140 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:28.140 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:28.141 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:28.144 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:28.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:28.437942 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:28.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:28.438198 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:29.357 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:29.358 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:29.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:29.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:29.360 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:29.360 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:29.361 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:29.361 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:29.363 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:29.441159 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:29.441392 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:29.465991 26723 master_runner.cc:186] Error getting master registration for 127.25.173.62:42167: Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111),
01:58:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:29.466429 26723 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:58:30.058 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:30.059 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:30.060 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:30.060 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:30.060 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:30.062 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:30.062 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:30.062 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:30.065 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:30.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:30.444058 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:30.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:30.444288 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:31.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:31.285568 26833 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:58:31.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:31.286022 26833 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } } peers { permanent_uuid: "941f53d6fb20487381a58726d84a3496" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } } peers { permanent_uuid: "ab1bf874594a456993831d9a1e862f26" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } }
01:58:31.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:31.287782 26833 leader_election.cc:288] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167), 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197)
01:58:31.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:31.289989 26744 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111) [suppressed 5 similar messages]
01:58:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:31.292930 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167): Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111)
01:58:31.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:31.293548 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197): Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:31.293824 26744 leader_election.cc:302] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab1bf874594a456993831d9a1e862f26; no voters: 941f53d6fb20487381a58726d84a3496, f0776e84fabb434d8af2316722bc7766
01:58:31.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:31.294289 26833 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:58:31.357 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:31.358 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:31.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:31.360 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:31.360 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:31.363 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:31.363 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:31.363 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:31.364 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:31.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:31.448006 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:31.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:31.448311 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:32.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:32.452260 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:32.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:32.452612 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:33.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:33.457414 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:33.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:33.457829 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:34 dist-test-slave-52vw krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387914, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:58:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:34.134413 26737 init.cc:295] Successfully reacquired a new kerberos TGT
01:58:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:34.461632 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:34.461959 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:34.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:34.469862 26723 master_runner.cc:186] Error getting master registration for 127.25.173.62:42167: Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111),
01:58:34.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:34.470228 26723 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:58:34.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:34.904126 26136 debug-util.cc:398] Leaking SignalData structure 0x7b08000af3c0 after lost signal to thread 26011
01:58:34.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:34.905138 26136 debug-util.cc:398] Leaking SignalData structure 0x7b08000af9e0 after lost signal to thread 26139
01:58:34.917 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:34.918 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:34.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:34.920 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:34.920 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:34.922 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:34.922 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:34.922 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:34.925 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:35.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:35.466300 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:35.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:35.466643 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:36.298 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:36.299 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:36.301 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:36.301 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:36.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:36.301 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:36.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:36.302 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:36.305 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:36.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:36.469782 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:36.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:36.470057 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:36.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:36.841473 26842 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:58:36.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:36.841969 26842 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } } peers { permanent_uuid: "941f53d6fb20487381a58726d84a3496" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } } peers { permanent_uuid: "ab1bf874594a456993831d9a1e862f26" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } }
01:58:36.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:36.843609 26842 leader_election.cc:288] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167), 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197)
01:58:36.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:36.846112 26744 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111) [suppressed 1 similar messages]
01:58:36.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:36.849287 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167): Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111)
01:58:36.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:36.849989 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197): Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:36.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:36.850310 26744 leader_election.cc:302] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab1bf874594a456993831d9a1e862f26; no voters: 941f53d6fb20487381a58726d84a3496, f0776e84fabb434d8af2316722bc7766
01:58:36.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:36.851007 26842 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:58:37.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:37.472720 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:37.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:37.473040 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:38.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:38.475795 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:38.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:38.476042 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:39.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:39.473002 26723 master_runner.cc:186] Error getting master registration for 127.25.173.62:42167: Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111),
01:58:39.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:39.473327 26723 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:58:39.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:39.478730 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:39.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:39.479023 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:40.238 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:40.239 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:40.240 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:40.240 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:40.241 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:40.242 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:40.242 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:40.242 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:40.244 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:40.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:40.482069 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:40.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:40.482355 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:41.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:41.485190 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:41.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:41.485464 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:42.488188 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:42.488481 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:43.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:43.492033 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:43.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:43.492374 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:43.557 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:43.559 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:43.560 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:43.561 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:43.562 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:43.562 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:43.562 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:43.562 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:43.565 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:43.977 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:43.978 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:43.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:43.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:43.980 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:43.981 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:43.981 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:43.982 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:43.983 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:44.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 01:58:44 dist-test-slave-52vw krb5kdc[26304](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790387924, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.173.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:58:44.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:44.137055 26737 init.cc:295] Successfully reacquired a new kerberos TGT
01:58:44.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:44.339242 26857 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:58:44.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:44.339841 26857 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } } peers { permanent_uuid: "941f53d6fb20487381a58726d84a3496" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } } peers { permanent_uuid: "ab1bf874594a456993831d9a1e862f26" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } }
01:58:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:44.341912 26857 leader_election.cc:288] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167), 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197)
01:58:44.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:44.344486 26744 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111) [suppressed 1 similar messages]
01:58:44.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:44.347429 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167): Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111)
01:58:44.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:44.347915 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197): Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:44.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:44.348299 26744 leader_election.cc:302] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab1bf874594a456993831d9a1e862f26; no voters: 941f53d6fb20487381a58726d84a3496, f0776e84fabb434d8af2316722bc7766
01:58:44.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:44.349054 26857 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:58:44.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:44.475208 26723 master_runner.cc:186] Error getting master registration for 127.25.173.62:42167: Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111),
01:58:44.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:44.475576 26723 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:58:44.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:44.495244 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:44.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:44.495493 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:45.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:45.498279 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:45.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:45.498517 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:46.377 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:46.378 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:46.379 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:46.379 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:46.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:46.380 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:46.380 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:46.380 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:46.382 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:46.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:46.501443 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:46.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:46.501689 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:46.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:46.811776 25995 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6d00 after lost signal to thread 25867
01:58:46.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:46.812538 25995 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba9c0 after lost signal to thread 25998
01:58:47.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:47.504544 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:47.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:47.504818 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:48.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:48.507462 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:48.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:48.507747 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:49.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:49.478042 26723 master_runner.cc:186] Error getting master registration for 127.25.173.62:42167: Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111),
01:58:49.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:49.478354 26723 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:58:49.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:49.510475 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:49.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:49.510772 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:50.358 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:50.359 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:50.359 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:50.359 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:50.360 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:50.360 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:50.360 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:50.360 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:50.363 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:50.513527 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:50.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:50.513813 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:51.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:51.516494 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:51.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:51.516805 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:51.578 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:51.579 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:51.579 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:51.579 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:51.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:51.580 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:51.580 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:51.580 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:51.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:52.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:52.452440 26872 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:58:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:52.452905 26872 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f0776e84fabb434d8af2316722bc7766" member_type: VOTER last_known_addr { host: "127.25.173.62" port: 42167 } } peers { permanent_uuid: "941f53d6fb20487381a58726d84a3496" member_type: VOTER last_known_addr { host: "127.25.173.61" port: 43197 } } peers { permanent_uuid: "ab1bf874594a456993831d9a1e862f26" member_type: VOTER last_known_addr { host: "127.25.173.60" port: 33419 } }
01:58:52.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:52.454814 26872 leader_election.cc:288] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167), 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197)
01:58:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:52.457270 26744 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111) [suppressed 1 similar messages]
01:58:52.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:52.460249 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer f0776e84fabb434d8af2316722bc7766 (127.25.173.62:42167): Network error: Client connection negotiation failed: client connection to 127.25.173.62:42167: connect: Connection refused (error 111)
01:58:52.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:52.460919 26744 leader_election.cc:334] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 941f53d6fb20487381a58726d84a3496 (127.25.173.61:43197): Network error: Client connection negotiation failed: client connection to 127.25.173.61:43197: connect: Connection refused (error 111)
01:58:52.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:52.461200 26744 leader_election.cc:302] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab1bf874594a456993831d9a1e862f26; no voters: 941f53d6fb20487381a58726d84a3496, f0776e84fabb434d8af2316722bc7766
01:58:52.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:52.461771 26872 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:58:52.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:52.519624 26809 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab1bf874594a456993831d9a1e862f26: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:58:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 01:58:52.519868 26809 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:58:52.677 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.62:42167(127.25.173.62:42167)] connecting to peer
01:58:52.678 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.173.61:43197(127.25.173.61:43197)] connecting to peer
01:58:52.679 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:52.679 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.173.62:42167(127.25.173.62:42167)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:52.679 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.62:42167: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167
01:58:52.679 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:52.680 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.61:43197(127.25.173.61:43197)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:52.680 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.173.61:43197: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
01:58:52.683 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419; will retry
01:58:52.688 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28387), Traces: [0ms] refreshing cache from master, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [199ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [199ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [602ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [617ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [646ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [649ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [651ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [653ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [681ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [684ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [686ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [701ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [703ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [705ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [706ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [717ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [724ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [726ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [728ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [780ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [783ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [784ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [817ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [824ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [826ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [826ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [923ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [926ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [927ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [931ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [933ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1117ms] refreshing cache from master, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1127ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [1130ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1337ms] refreshing cache from master, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [1343ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [2237ms] refreshing cache from master, [2237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [2238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [2239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [2241ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [2243ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [2243ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [3837ms] refreshing cache from master, [3837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [3838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [3840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [3840ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [3840ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [3844ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [3845ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [5056ms] refreshing cache from master, [5057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [5057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [5058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [5059ms] too many traces: truncated at 100 traces, deferred=Deferred@1628151087(state=PENDING, result=null, callback=(continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858), errback=(continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858)))
01:58:52.693 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.173.61:43197(127.25.173.61:43197)] disconnecting while in state TERMINATED
01:58:52.694 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.173.60:33419(127.25.173.60:33419)] disconnecting while in state READY
01:58:52.694 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.173.62:42167(127.25.173.62:42167)] disconnecting while in state TERMINATED
01:58:52.698 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:58:52.700 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:58:52.701 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.25.173.60:33419(127.25.173.60:33419)] handling channelInactive
01:58:52.702 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.173.60:33419(127.25.173.60:33419)] cleaning up while in state READY due to: connection closed
01:58:52.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:52.706780 26292 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu with pid 26723
01:58:53.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T01:58:53Z chronyd exiting
01:58:54.931 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:58:54.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:54.933022 25568 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu with pid 25847
01:58:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:54.960453 25568 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu with pid 26010
01:58:54.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:54.986492 25568 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu with pid 26150
01:58:55.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:55.013489 25568 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu with pid 25580
01:58:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:55.044483 25568 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu with pid 25650
01:58:55.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 01:58:55.073562 25568 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskaDlrHb/build/tsan/bin/kudu with pid 25725
01:58:56.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T01:58:56Z chronyd exiting
01:58:57.249 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28387), Traces: [0ms] refreshing cache from master, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [199ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [199ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [602ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [617ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [646ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [649ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [651ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [653ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [681ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [684ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [686ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [701ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [703ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [705ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [706ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [717ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [724ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [726ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [728ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [780ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [783ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [784ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [817ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [824ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [826ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [826ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [923ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [926ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [927ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [931ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [933ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1117ms] refreshing cache from master, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1127ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [1130ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1337ms] refreshing cache from master, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [1343ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [2237ms] refreshing cache from master, [2237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [2238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [2239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [2241ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [2243ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [2243ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [3837ms] refreshing cache from master, [3837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [3838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [3840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [3840ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [3840ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [3844ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [3845ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [5056ms] refreshing cache from master, [5057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [5057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [5058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [5059ms] too many traces: truncated at 100 traces, deferred=Deferred@1628151087(state=PENDING, result=null, callback=(continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858), errback=(continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.173.61:43197
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28387), Traces: [0ms] refreshing cache from master, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [199ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [199ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [602ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [617ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [646ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [649ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [651ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [653ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [681ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [684ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [686ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [701ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [703ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [705ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [706ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [717ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [724ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [726ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [728ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [780ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [783ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [784ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [817ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [824ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [826ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [826ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [923ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [926ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [927ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [931ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [933ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1117ms] refreshing cache from master, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1127ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [1130ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1337ms] refreshing cache from master, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [1343ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [2237ms] refreshing cache from master, [2237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [2238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [2239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [2241ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [2243ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [2243ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [3837ms] refreshing cache from master, [3837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [3838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [3840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [3840ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [3840ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [3844ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [3845ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [5056ms] refreshing cache from master, [5057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [5057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [5058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [5059ms] too many traces: truncated at 100 traces, deferred=Deferred@1628151087(state=PENDING, result=null, callback=(continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858), errback=(continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.173.61:43197
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
01:58:57.993 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
01:58:57.994 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28387), Traces: [0ms] refreshing cache from master, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [199ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [199ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [602ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [617ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [646ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [649ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [651ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [653ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [681ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [684ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [686ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [701ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [703ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [705ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [706ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [717ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [724ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [726ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [728ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [780ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [783ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [784ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [817ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [824ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [826ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [826ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [923ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [926ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [927ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [931ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [933ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1117ms] refreshing cache from master, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1127ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [1130ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1337ms] refreshing cache from master, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [1343ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [2237ms] refreshing cache from master, [2237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [2238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [2239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [2241ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [2243ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [2243ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [3837ms] refreshing cache from master, [3837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [3838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [3840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [3840ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [3840ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [3844ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [3845ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [5056ms] refreshing cache from master, [5057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [5057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [5058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [5059ms] too many traces: truncated at 100 traces, deferred=Deferred@1628151087(state=PENDING, result=null, callback=(continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858), errback=(continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.173.61:43197
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28387), Traces: [0ms] refreshing cache from master, [29ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [199ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [199ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [602ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [617ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [646ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [649ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [651ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [653ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [681ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [684ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [686ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [687ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [701ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [703ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [705ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [706ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [717ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [724ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [726ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [728ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [780ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [783ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [784ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [817ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [820ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [824ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [826ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [826ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [923ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [926ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [927ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [931ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [933ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1117ms] refreshing cache from master, [1118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [1121ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [1125ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1127ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [1130ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1337ms] refreshing cache from master, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [1343ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [2237ms] refreshing cache from master, [2237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [2238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [2239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [2241ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [2243ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [2243ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [3837ms] refreshing cache from master, [3837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [3838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [3840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [3840ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.62:42167: Network error: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167, [3840ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.61:43197: Network error: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [3844ms] Sub RPC ConnectToMaster: received response from server master-127.25.173.60:33419: OK, [3845ms] delaying RPC due to: Service unavailable: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197, [5056ms] refreshing cache from master, [5057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.62:42167, [5057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.61:43197, [5058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.173.60:33419, [5059ms] too many traces: truncated at 100 traces, deferred=Deferred@1628151087(state=PENDING, result=null, callback=(continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858), errback=(continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068) -> (continuation of Deferred@1876238217 after retry RPC after error@1356758715) -> (continuation of Deferred@527989985 after retry RPC after error@297601166) -> (continuation of Deferred@1448309863 after retry RPC after error@1977344944) -> (continuation of Deferred@689440414 after retry RPC after error@1343783499) -> (continuation of Deferred@497362019 after retry RPC after error@935956996) -> (continuation of Deferred@1577864853 after retry RPC after error@1242727877) -> (continuation of Deferred@2116036312 after retry RPC after error@84066986) -> (continuation of Deferred@969301113 after retry RPC after error@1738857001) -> (continuation of Deferred@416102115 after retry RPC after error@197763266) -> (continuation of Deferred@6750294 after retry RPC after error@1104406497) -> (continuation of Deferred@811117334 after retry RPC after error@1985073775) -> (continuation of Deferred@1309415747 after retry RPC after error@280888204) -> (continuation of Deferred@1699416726 after retry RPC after error@502361850) -> (continuation of Deferred@865708550 after retry RPC after error@1398504858)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.173.62:42167,127.25.173.61:43197,127.25.173.60:33419) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.62:42167(127.25.173.62:42167): Connection refused: /127.25.173.62:42167,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.173.61:43197(127.25.173.61:43197): Connection refused: /127.25.173.61:43197
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.173.61:43197
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)